diff options
438 files changed, 11078 insertions, 6259 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index ac39fe235b..0c43e61592 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -39,6 +39,7 @@ after_build: - ps: clcache -z before_test: - ps: ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in"))) +- ps: ${conf_ini} = ${conf_ini}.Replace("@PACKAGE_NAME@", "Bitcoin Core") - ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER}) - ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER}) - ps: ${conf_ini} = ${conf_ini}.Replace("@EXEEXT@", ".exe") diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..9104a0a3d1 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,26 @@ +task: + name: "FreeBsd 12.0 amd64 [GOAL: install] [no depends, only system libs]" + freebsd_instance: + image: freebsd-12-0-release-amd64 + ccache_cache: + folder: "/tmp/ccache_dir" + env: + MAKEJOBS: "-j3" + CONFIGURE_OPTS: "--disable-dependency-tracking" + GOAL: "install" + CCACHE_SIZE: "200M" + CCACHE_COMPRESS: 1 + CCACHE_DIR: "/tmp/ccache_dir" + install_script: + - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache + - ./contrib/install_db4.sh $(pwd) + - ccache --max-size=${CCACHE_SIZE} + configure_script: + - ./autogen.sh + - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false) + make_script: + - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false ) + check_script: + - gmake check ${MAKEJOBS} VERBOSE=1 + functional_test_script: + - ./test/functional/test_runner.py --ci --combinedlogslen=1000 --quiet --failfast diff --git a/.gitignore b/.gitignore index 994d67402f..be784024a0 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ src/qt/bitcoin-qt.includes *.a *.pb.cc *.pb.h +*.dat *.log *.trs @@ -120,3 +121,6 @@ contrib/devtools/split-debug.sh # Output from running db4 installation db4/ + +# clang-check +*.plist diff --git a/.python-version b/.python-version index 7bcbb3808b..c49282585a 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.4.9 +3.5.6 diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000..69d8c6aee4 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,261 @@ +[style] +# Align closing bracket with visual indentation. +align_closing_bracket_with_visual_indent=True + +# Allow dictionary keys to exist on multiple lines. For example: +# +# x = { +# ('this is the first element of a tuple', +# 'this is the second element of a tuple'): +# value, +# } +allow_multiline_dictionary_keys=False + +# Allow lambdas to be formatted on more than one line. +allow_multiline_lambdas=False + +# Allow splits before the dictionary value. +allow_split_before_dict_value=True + +# Number of blank lines surrounding top-level function and class +# definitions. +blank_lines_around_top_level_definition=2 + +# Insert a blank line before a class-level docstring. +blank_line_before_class_docstring=False + +# Insert a blank line before a module docstring. +blank_line_before_module_docstring=False + +# Insert a blank line before a 'def' or 'class' immediately nested +# within another 'def' or 'class'. For example: +# +# class Foo: +# # <------ this blank line +# def method(): +# ... +blank_line_before_nested_class_or_def=False + +# Do not split consecutive brackets. Only relevant when +# dedent_closing_brackets is set. For example: +# +# call_func_that_takes_a_dict( +# { +# 'key1': 'value1', +# 'key2': 'value2', +# } +# ) +# +# would reformat to: +# +# call_func_that_takes_a_dict({ +# 'key1': 'value1', +# 'key2': 'value2', +# }) +coalesce_brackets=False + +# The column limit. +column_limit=160 + +# The style for continuation alignment. Possible values are: +# +# - SPACE: Use spaces for continuation alignment. This is default behavior. +# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns +# (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation +# alignment. +# - LESS: Slightly left if cannot vertically align continuation lines with +# indent characters. +# - VALIGN-RIGHT: Vertically align continuation lines with indent +# characters. Slightly right (one more indent character) if cannot +# vertically align continuation lines with indent characters. +# +# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is +# enabled. +continuation_align_style=SPACE + +# Indent width used for line continuations. +continuation_indent_width=4 + +# Put closing brackets on a separate line, dedented, if the bracketed +# expression can't fit in a single line. Applies to all kinds of brackets, +# including function definitions and calls. For example: +# +# config = { +# 'key1': 'value1', +# 'key2': 'value2', +# } # <--- this bracket is dedented and on a separate line +# +# time_series = self.remote_client.query_entity_counters( +# entity='dev3246.region1', +# key='dns.query_latency_tcp', +# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), +# start_ts=now()-timedelta(days=3), +# end_ts=now(), +# ) # <--- this bracket is dedented and on a separate line +dedent_closing_brackets=False + +# Disable the heuristic which places each list element on a separate line +# if the list is comma-terminated. +disable_ending_comma_heuristic=False + +# Place each dictionary entry onto its own line. +each_dict_entry_on_separate_line=True + +# The regex for an i18n comment. The presence of this comment stops +# reformatting of that line, because the comments are required to be +# next to the string they translate. +i18n_comment= + +# The i18n function call names. The presence of this function stops +# reformattting on that line, because the string it has cannot be moved +# away from the i18n comment. +i18n_function_call= + +# Indent the dictionary value if it cannot fit on the same line as the +# dictionary key. For example: +# +# config = { +# 'key1': +# 'value1', +# 'key2': value1 + +# value2, +# } +indent_dictionary_value=False + +# The number of columns to use for indentation. +indent_width=4 + +# Join short lines into one line. E.g., single line 'if' statements. +join_multiple_lines=True + +# Do not include spaces around selected binary operators. For example: +# +# 1 + 2 * 3 - 4 / 5 +# +# will be formatted as follows when configured with "*,/": +# +# 1 + 2*3 - 4/5 +# +no_spaces_around_selected_binary_operators= + +# Use spaces around default or named assigns. +spaces_around_default_or_named_assign=False + +# Use spaces around the power operator. +spaces_around_power_operator=False + +# The number of spaces required before a trailing comment. +spaces_before_comment=2 + +# Insert a space between the ending comma and closing bracket of a list, +# etc. +space_between_ending_comma_and_closing_bracket=True + +# Split before arguments +split_all_comma_separated_values=False + +# Split before arguments if the argument list is terminated by a +# comma. +split_arguments_when_comma_terminated=False + +# Set to True to prefer splitting before '&', '|' or '^' rather than +# after. +split_before_bitwise_operator=True + +# Split before the closing bracket if a list or dict literal doesn't fit on +# a single line. +split_before_closing_bracket=True + +# Split before a dictionary or set generator (comp_for). For example, note +# the split before the 'for': +# +# foo = { +# variable: 'Hello world, have a nice day!' +# for variable in bar if variable != 42 +# } +split_before_dict_set_generator=True + +# Split before the '.' if we need to split a longer expression: +# +# foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) +# +# would reformat to something like: +# +# foo = ('This is a really long string: {}, {}, {}, {}' +# .format(a, b, c, d)) +split_before_dot=False + +# Split after the opening paren which surrounds an expression if it doesn't +# fit on a single line. +split_before_expression_after_opening_paren=False + +# If an argument / parameter list is going to be split, then split before +# the first argument. +split_before_first_argument=False + +# Set to True to prefer splitting before 'and' or 'or' rather than +# after. +split_before_logical_operator=True + +# Split named assignments onto individual lines. +split_before_named_assigns=True + +# Set to True to split list comprehensions and generators that have +# non-trivial expressions and multiple clauses before each of these +# clauses. For example: +# +# result = [ +# a_long_var + 100 for a_long_var in xrange(1000) +# if a_long_var % 10] +# +# would reformat to something like: +# +# result = [ +# a_long_var + 100 +# for a_long_var in xrange(1000) +# if a_long_var % 10] +split_complex_comprehension=False + +# The penalty for splitting right after the opening bracket. +split_penalty_after_opening_bracket=30 + +# The penalty for splitting the line after a unary operator. +split_penalty_after_unary_operator=10000 + +# The penalty for splitting right before an if expression. +split_penalty_before_if_expr=0 + +# The penalty of splitting the line around the '&', '|', and '^' +# operators. +split_penalty_bitwise_operator=300 + +# The penalty for splitting a list comprehension or generator +# expression. +split_penalty_comprehension=80 + +# The penalty for characters over the column limit. +split_penalty_excess_character=7000 + +# The penalty incurred by adding a line split to the unwrapped line. The +# more line splits added the higher the penalty. +split_penalty_for_added_line_split=30 + +# The penalty of splitting a list of "import as" names. For example: +# +# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, +# long_argument_2, +# long_argument_3) +# +# would reformat to something like: +# +# from a_very_long_or_indented_module_name_yada_yad import ( +# long_argument_1, long_argument_2, long_argument_3) +split_penalty_import_names=0 + +# The penalty of splitting the line around the 'and' and 'or' +# operators. +split_penalty_logical_operator=300 + +# Use the Tab character for indentation. +use_tabs=False + diff --git a/.travis.yml b/.travis.yml index 0ede31b4a6..21d1062c26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,30 @@ +# The test build matrix (stage: test) is constructed to test a wide range of +# configurations, rather than a single pass/fail. This helps to catch build +# failures and logic errors that present on platforms other than the ones the +# author has tested. +# +# Some builders use the dependency-generator in `./depends`, rather than using +# apt-get to install build dependencies. This guarantees that the tester is +# using the same versions as Gitian, so the build results are nearly identical +# to what would be found in a final release. +# +# In order to avoid rebuilding all dependencies for each build, the binaries +# are cached and re-used when possible. Changes in the dependency-generator +# will trigger cache-invalidation and rebuilds as necessary. +# +# These caches can be manually removed if necessary. This is one of the very +# few manual operations that is possible with Travis, and it can be done by a +# Bitcoin Core GitHub member via the Travis web interface [0]. +# +# Travis CI uploads the cache after the script phase of the build [1]. +# However, the build is terminated without saving the chache if it takes over +# 50 minutes [2]. Thus, if we spent too much time in early build stages, fail +# with an error and save the cache. +# +# [0] https://travis-ci.org/bitcoin/bitcoin/caches +# [1] https://docs.travis-ci.com/user/caching/#build-phases +# [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts + dist: xenial os: linux language: minimal @@ -26,6 +53,7 @@ env: - SDK_URL=https://bitcoincore.org/depends-sources/sdks - WINEDEBUG=fixme-all - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source .travis/test_03_before_install.sh install: @@ -33,8 +61,11 @@ install: before_script: - set -o errexit; source .travis/test_05_before_script.sh script: - - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script_a.sh; fi - - if [ $SECONDS -gt 1800 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script_b.sh; fi + - export CONTINUE=1 + - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long + - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left + - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG @@ -46,7 +77,7 @@ jobs: env: cache: false language: python - python: '3.4' # Oldest supported version according to doc/dependencies.md + python: '3.5' # Oldest supported version according to doc/dependencies.md install: - set -o errexit; source .travis/lint_04_install.sh before_script: @@ -158,7 +189,7 @@ jobs: name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' env: >- HOST=x86_64-apple-darwin14 - PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" OSX_SDK=10.11 RUN_UNIT_TESTS=false RUN_FUNCTIONAL_TESTS=false diff --git a/.travis/lint_06_script.sh b/.travis/lint_06_script.sh index 701e6d8005..eeebc80ec0 100755 --- a/.travis/lint_06_script.sh +++ b/.travis/lint_06_script.sh @@ -20,6 +20,6 @@ test/lint/lint-all.sh if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys && - travis_wait 50 contrib/verify-commits/verify-commits.py --clean-merge=2; + travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && + ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 139fe7dc52..007ebd7ccf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,7 +62,7 @@ Commit messages should be verbose by default consisting of a short subject line paragraph(s), unless the title alone is self-explanatory (like "Corrected typo in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for -your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). +your decisions. Further explanation [here](https://chris.beams.io/posts/git-commit/). If a particular commit references another issue, please add the reference. For example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords @@ -239,7 +239,10 @@ consensus to merge a pull request (remember that discussions may have been spread out over GitHub, mailing list and IRC discussions). The following language is used within pull-request comments: - - ACK means "I have tested the code and I agree it should be merged"; + - (t)ACK means "I have tested the code and I agree it should be merged", involving + change-specific manual testing in addition to running the unit and functional + tests, and in case it is not obvious how the manual testing was done, it should + be described; - NACK means "I disagree this should be merged", and must be accompanied by sound technical justification (or in certain cases of copyright/patent/licensing issues, legal justification). NACKs without accompanying reasoning may be diff --git a/Makefile.am b/Makefile.am index 85674f819a..ec0743c3fa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,6 +43,7 @@ DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ + $(top_srcdir)/contrib/debian/copyright \ $(top_srcdir)/contrib/init \ $(top_srcdir)/contrib/install_db4.sh DIST_SHARE = \ @@ -66,7 +66,7 @@ Translations ------------ Changes to translations as well as new translations can be submitted to -[Bitcoin Core's Transifex page](https://www.transifex.com/projects/p/bitcoin/). +[Bitcoin Core's Transifex page](https://www.transifex.com/bitcoin/bitcoin/). Translations are periodically pulled from Transifex and merged into the git repository. See the [translation process](doc/translation_process.md) for details on how this works. diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj index 368fe7ead5..723e230d3a 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj @@ -20,6 +20,10 @@ </ProjectConfiguration> </ItemGroup> <ItemGroup> + <ClCompile Include="..\..\src\test\util.h" /> + <ClCompile Include="..\..\src\test\util.cpp" /> + <ClCompile Include="..\..\src\test\setup_common.h" /> + <ClCompile Include="..\..\src\test\setup_common.cpp" /> <ClCompile Include="..\..\src\bench\base58.cpp" /> <ClCompile Include="..\..\src\bench\bech32.cpp" /> <ClCompile Include="..\..\src\bench\bench.cpp" /> @@ -32,8 +36,10 @@ <ClCompile Include="..\..\src\bench\examples.cpp" /> <ClCompile Include="..\..\src\bench\lockedpool.cpp" /> <ClCompile Include="..\..\src\bench\mempool_eviction.cpp" /> + <ClCompile Include="..\..\src\bench\rpc_mempool.cpp" /> <ClCompile Include="..\..\src\bench\merkle_root.cpp" /> <ClCompile Include="..\..\src\bench\rollingbloom.cpp" /> + <ClCompile Include="..\..\src\bench\wallet_balance.cpp" /> <ClCompile Include="..\..\src\bench\verify_script.cpp" /> </ItemGroup> <ItemGroup> diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 4ac27dae3f..b5a05e2629 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -11,10 +11,10 @@ #define CLIENT_VERSION_IS_RELEASE false /* Major version */ -#define CLIENT_VERSION_MAJOR 1 +#define CLIENT_VERSION_MAJOR 0 /* Minor version */ -#define CLIENT_VERSION_MINOR 17 +#define CLIENT_VERSION_MINOR 18 /* Build revision */ #define CLIENT_VERSION_REVISION 99 @@ -29,7 +29,7 @@ #define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" /* Copyright year */ -#define COPYRIGHT_YEAR 2018 +#define COPYRIGHT_YEAR 2019 /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in index 48a117bcfe..0b4aa1e407 100644 --- a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in +++ b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in @@ -151,6 +151,9 @@ </ItemDefinitionGroup> <ItemGroup> @SOURCE_FILES@ + <ClCompile Include="..\..\src\wallet\init.cpp"> + <ObjectFileName>$(IntDir)wallet_init.obj</ObjectFileName> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="packages.config" /> diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj index a32aafbd74..227b1ebcd2 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj @@ -22,6 +22,7 @@ <ItemGroup> <ClCompile Include="..\..\src\arith_uint256.cpp" /> <ClCompile Include="..\..\src\consensus\merkle.cpp" /> + <ClCompile Include="..\..\src\consensus\tx_check.cpp" /> <ClCompile Include="..\..\src\crypto\aes.cpp" /> <ClCompile Include="..\..\src\crypto\chacha20.cpp" /> <ClCompile Include="..\..\src\crypto\hmac_sha256.cpp" /> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index d05a03699f..92fe95bddc 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -24,8 +24,8 @@ <ClCompile Include="..\..\src\test\*_properties.cpp" /> <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> - <ClCompile Include="..\..\src\test\test_bitcoin.cpp" /> - <ClCompile Include="..\..\src\test\test_bitcoin_main.cpp" /> + <ClCompile Include="..\..\src\test\setup_common.cpp" /> + <ClCompile Include="..\..\src\test\main.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> </ItemGroup> <ItemGroup> diff --git a/configure.ac b/configure.ac index dec6d4fac3..854d6b1d49 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 17) +define(_CLIENT_VERSION_MINOR, 18) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -85,8 +85,8 @@ AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.4 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.4 python3.5 python3.6 python3.7 python3 python]) +dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) @@ -195,8 +195,8 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=no]) AC_ARG_ENABLE([asm], - [AS_HELP_STRING([--enable-asm], - [Enable assembly routines (default is yes)])], + [AS_HELP_STRING([--disable-asm], + [disable assembly routines (enabled by default)])], [use_asm=$enableval], [use_asm=yes]) @@ -339,6 +339,13 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi +enable_hwcrc32=no +enable_sse41=no +enable_avx2=no +enable_shani=no + +if test "x$use_asm" = "xyes"; then + # Check for optional instruction set support. Enabling these does _not_ imply that all code will # be compiled with them, rather that specific objects/libs may use them after checking for runtime # compatibility. @@ -416,6 +423,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ) CXXFLAGS="$TEMP_CXXFLAGS" +fi + CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], @@ -827,8 +836,23 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ } ])], [ - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) + case $host in + *mingw*) + # mingw32's implementation of thread_local has also been shown to behave + # erroneously under concurrent usage; see: + # https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 + AC_MSG_RESULT(no) + ;; + *darwin*) + # TODO enable thread_local on later versions of Darwin where it is + # supported (per https://stackoverflow.com/a/29929949) + AC_MSG_RESULT(no) + ;; + *) + AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) + AC_MSG_RESULT(yes) + ;; + esac ], [ AC_MSG_RESULT(no) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 6ee65f40be..0c8c396503 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -7,6 +7,8 @@ clang-format-diff.py A script to format unified git diffs according to [.clang-format](../../src/.clang-format). +Requires `clang-format`, installed e.g. via `brew install clang-format` on macOS. + For instance, to format the last commit with 0 lines of context, the script should be called from the git root folder as follows. @@ -167,7 +169,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage after a gitian build: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py If only supported symbols are used the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index abfa5ed5ae..2e4657f1dd 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -8,9 +8,18 @@ MAPPING = { 'core_write.cpp': 'core_io.cpp', } +# Directories with header-based modules, where the assumption that .cpp files +# define functions and variables declared in corresponding .h files is +# incorrect. +HEADER_MODULE_PATHS = [ + 'interfaces/' +] + def module_name(path): if path in MAPPING: path = MAPPING[path] + if any(path.startswith(dirpath) for dirpath in HEADER_MODULE_PATHS): + return path if path.endswith(".h"): return path[:-2] if path.endswith(".c"): diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index ab834b7623..a57ecf9818 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -47,18 +47,36 @@ def git_config_get(option, default=None): except subprocess.CalledProcessError: return default -def retrieve_pr_info(repo,pull,ghtoken): +def get_response(req_url, ghtoken): + req = Request(req_url) + if ghtoken is not None: + req.add_header('Authorization', 'token ' + ghtoken) + return urlopen(req) + +def retrieve_json(req_url, ghtoken, use_pagination=False): ''' - Retrieve pull request information from github. - Return None if no title can be found, or an error happens. + Retrieve json from github. + Return None if an error happens. ''' try: - req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) - if ghtoken is not None: - req.add_header('Authorization', 'token ' + ghtoken) - result = urlopen(req) reader = codecs.getreader('utf-8') - obj = json.load(reader(result)) + if not use_pagination: + return json.load(reader(get_response(req_url, ghtoken))) + + obj = [] + page_num = 1 + while True: + req_url_page = '{}?page={}'.format(req_url, page_num) + result = get_response(req_url_page, ghtoken) + obj.extend(json.load(reader(result))) + + link = result.headers.get('link', None) + if link is not None: + link_next = [l for l in link.split(',') if 'rel="next"' in l] + if len(link_next) > 0: + page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")]) + continue + break return obj except HTTPError as e: error_message = e.read() @@ -69,6 +87,18 @@ def retrieve_pr_info(repo,pull,ghtoken): print('Warning: unable to retrieve pull information from github: %s' % e) return None +def retrieve_pr_info(repo,pull,ghtoken): + req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull + return retrieve_json(req_url,ghtoken) + +def retrieve_pr_comments(repo,pull,ghtoken): + req_url = "https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments" + return retrieve_json(req_url,ghtoken,use_pagination=True) + +def retrieve_pr_reviews(repo,pull,ghtoken): + req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews" + return retrieve_json(req_url,ghtoken,use_pagination=True) + def ask_prompt(text): print(text,end=" ",file=stderr) stderr.flush() @@ -133,6 +163,16 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def get_acks_from_comments(head_commit, comments): + assert len(head_commit) == 6 + ack_str ='\n\nACKs for commit {}:\n'.format(head_commit) + for c in comments: + review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_commit in l] + if review: + ack_str += ' {}:\n'.format(c['user']['login']) + ack_str += ' {}\n'.format(review[0]) + return ack_str + def print_merge_details(pull, title, branch, base_branch, head_branch): print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) @@ -185,6 +225,9 @@ def main(): info = retrieve_pr_info(repo,pull,ghtoken) if info is None: sys.exit(1) + comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) + if comments is None: + sys.exit(1) title = info['title'].strip() body = info['body'].strip() # precedence order for destination branch argument: @@ -238,6 +281,7 @@ def main(): message = firstline + '\n\n' message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' + message += get_acks_from_comments(head_commit=subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8')[:6], comments=comments) try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) except subprocess.CalledProcessError: diff --git a/contrib/devtools/split-debug.sh.in b/contrib/devtools/split-debug.sh.in index deda49cc54..92b72b1446 100644 --- a/contrib/devtools/split-debug.sh.in +++ b/contrib/devtools/split-debug.sh.in @@ -1,5 +1,5 @@ #!/bin/sh - +set -e if [ $# -ne 3 ]; then echo "usage: $0 <input> <stripped-binary> <debug-binary>" fi diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c6158c9422..7729dd7257 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -9,7 +9,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import re diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000..16d03e1fff --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) + "denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256()); + "fs_tests/fsbridge_fstream" # deterministic test failure? + "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() + "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() + "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) +) + +TEST_BITCOIN_BINARY="src/test/test_bitcoin" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e97072c80a..5845d8fd89 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-linux-0.18" +name: "bitcoin-core-linux-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -30,12 +31,13 @@ packages: - "faketime" - "bsdmainutils" - "ca-certificates" -- "python" +- "python3" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" @@ -179,8 +181,8 @@ script: | find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg cp ../doc/README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 297a136fae..4cfca403b1 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,5 +1,6 @@ --- name: "bitcoin-dmg-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -12,6 +13,8 @@ remotes: files: - "bitcoin-osx-unsigned.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH=`pwd`:$PATH diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 87d8007ccb..24292d089a 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-osx-0.18" +name: "bitcoin-core-osx-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -23,9 +24,9 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" +- "python3" +- "python3-dev" +- "python3-setuptools" - "fonts-tuffy" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" @@ -33,6 +34,8 @@ remotes: files: - "MacOSX10.11.sdk.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin14" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 045be873e9..656c6d9b7a 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,5 +1,6 @@ --- name: "bitcoin-win-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -16,6 +17,8 @@ files: - "osslsigncode-Backports-to-1.7.1.patch" - "bitcoin-win-unsigned.tar.gz" script: | + set -e -o pipefail + BUILD_DIR=`pwd` SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 31b9c309c7..eca32a5dc5 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-win-0.18" +name: "bitcoin-core-win-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -20,13 +21,15 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" +- "python3" - "rename" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 80c18aa889..33f0f7e5b0 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -6,6 +6,7 @@ C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random +6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 2985106982..25a1c7351a 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,6 +1,5 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts -run using Python 3 but are compatible with Python 2. +Construct a linear, no-fork, best version of the Bitcoin blockchain. ## Step 1: Download hash list diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 56c1fbfc92..468aec04b5 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -16,7 +16,7 @@ import hashlib import datetime import time from collections import namedtuple -from binascii import hexlify, unhexlify +from binascii import unhexlify settings = {} @@ -61,7 +61,7 @@ def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') + hash_str = hash.hex() return hash_str def get_blk_dt(blk_hdr): @@ -213,7 +213,7 @@ class BlockDataCopier: inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) + print("Invalid magic: " + inMagic.hex()) return inLenLE = inhdr[4:] su = struct.unpack("<I", inLenLE) diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 6bbed01073..255ce75092 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -5,6 +5,7 @@ """Verify commits against a trusted keys list.""" import argparse import hashlib +import logging import os import subprocess import sys @@ -66,6 +67,11 @@ def tree_sha512sum(commit='HEAD'): return overall.hexdigest() def main(): + + # Enable debug logging if running in CI + if 'CI' in os.environ and os.environ['CI'].lower() == "true": + logging.getLogger().setLevel(logging.DEBUG) + # Parse arguments parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') @@ -95,6 +101,10 @@ def main(): # Iterate through commits while True: + + # Log a message to prevent Travis from timing out + logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8])) + if current_commit == verified_root: print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) sys.exit(0) diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert index 200b30a3f0..5bc5dc5809 100644 --- a/contrib/windeploy/win-codesign.cert +++ b/contrib/windeploy/win-codesign.cert @@ -1,99 +1,100 @@ -----BEGIN CERTIFICATE----- -MIIFTTCCBDWgAwIBAgIRALlW05RLwG2hMQMX5d/o5J8wDQYJKoZIhvcNAQELBQAw -fTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxIzAhBgNV -BAMTGkNPTU9ETyBSU0EgQ29kZSBTaWduaW5nIENBMB4XDTE2MDIwMzAwMDAwMFoX -DTE5MDMwNTIzNTk1OVowgbUxCzAJBgNVBAYTAlVTMQ4wDAYDVQQRDAU5ODEwNDEL -MAkGA1UECAwCV0ExEDAOBgNVBAcMB1NlYXR0bGUxEDAOBgNVBAkMB1N0ZSAzMDAx -FzAVBgNVBAkMDjcxIENvbHVtYmlhIFN0MSUwIwYDVQQKDBxUaGUgQml0Y29pbiBG -b3VuZGF0aW9uLCBJbmMuMSUwIwYDVQQDDBxUaGUgQml0Y29pbiBGb3VuZGF0aW9u -LCBJbmMuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw37Vrv9Gbku0 -+kuV0t89TuyxtAcmT7QE4GcwESKKjmkxfzD9a0qlhqk8GfQ+fw4DHNN+nLKNv7xB -bk6aS7J2v2DcXkOjrP99P9jqgTkp7MC04VtG3OqVRGB+gum0pptRovYZUQXIdkY7 -GJOok/NDagwKiiUe2V2meZ7UctsZNvYeilQdTgKIIhrMB9NowCOhT8ocVL4Ki55/ -l7hukJn3fueCM3fHTwY2/1gaGsOHoCkFRsD7vokjAVpiY+8rUgvHjb0gxgojiVGd -6a6/F5XJwKJacvUyN4Hfc2K5lRMQjTTmo4aWNWIa0iJ3TK9BHpdSLJBqerMPvmnM -kkapS+ZTNQIDAQABo4IBjTCCAYkwHwYDVR0jBBgwFoAUKZFg/4pN+uv5pmq4z/nm -S71JzhIwHQYDVR0OBBYEFONpQ+cV82URVe+V8G57377KxxexMA4GA1UdDwEB/wQE -AwIHgDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMBEGCWCGSAGG -+EIBAQQEAwIEEDBGBgNVHSAEPzA9MDsGDCsGAQQBsjEBAgEDAjArMCkGCCsGAQUF -BwIBFh1odHRwczovL3NlY3VyZS5jb21vZG8ubmV0L0NQUzBDBgNVHR8EPDA6MDig -NqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9SU0FDb2RlU2lnbmlu -Z0NBLmNybDB0BggrBgEFBQcBAQRoMGYwPgYIKwYBBQUHMAKGMmh0dHA6Ly9jcnQu -Y29tb2RvY2EuY29tL0NPTU9ET1JTQUNvZGVTaWduaW5nQ0EuY3J0MCQGCCsGAQUF -BzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB -AGnBSi9K/9rgTAyKFKrfGWSfNOwAghmsnsvpZSQ7QyoGWBFKSgCs/70kErl18oHA -g7Y8loQB1yukZmJaCa3OvGud7smn45TCh0TMf4EpP20Wxf4rMQTxwAatasHL3+vi -I+Nl5bsRZ09kWjvayqLII5upjS/yq0JfpmyGl5k2C/fIpztq0iOLvqWlXcL4+51r -cMUAfX6E6EaZQm//ikp+w2+7MEXTKguOuV3gwsrTy0DsvkZl4YDgx/FA4ImzXopv -d+3KJPLvO+OSBqUD3JPwXHnuJqGAbLBFyyCa/feGUjLlR8cxcNWLWdp4qxtoIUPG -3wTsC9YgrglS0F7FKMXlNRY= +MIIFcTCCBFmgAwIBAgIRALWcUnSOxv9FQW3xdaMDO6swDQYJKoZIhvcNAQELBQAw +fDELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMSQwIgYDVQQD +ExtTZWN0aWdvIFJTQSBDb2RlIFNpZ25pbmcgQ0EwHhcNMTkwMzI3MDAwMDAwWhcN +MjAwMzI2MjM1OTU5WjCBtDELMAkGA1UEBhMCQ0gxDTALBgNVBBEMBDgwMDUxCzAJ +BgNVBAgMAlpIMRAwDgYDVQQHDAdaw7xyaWNoMRcwFQYDVQQJDA5NYXR0ZW5nYXNz +ZSAyNzEuMCwGA1UECgwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjEuMCwGA1UEAwwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK04VDwiY1wxcW3E +WTTGmnbciCwETwC96DG4qcoH2PPNsVy3dfwGh0C02Qj2vL64IfwIGUFSgREvyjZk +CNhEuJO2e0nO0rKNNH5v/JO+P7/VYPZkF5a3uUz9ulmihULXioieHB/q0l6BmiJL ++cYaMVfidL9Y+IJwgiTqjnpRhv1Ik083SPsu6GcfQT9MJfY/+xse2EP0l4GfdFE6 +DRcWjiC8UHpfpGYcImzSFZZpbFbqoAyhueCl28QU4f8QAbS6BqNfaAK9MMACWDcK +eTz3C5JK6CiUxOnGIxilXhljuybFUjR4jGl5eTRpuPWk95NTTYS36q+bx/1nYelx +0n4nnDMCAwEAAaOCAbMwggGvMB8GA1UdIwQYMBaAFA7hOqhTOjHVir7Bu61nGgOF +rTQOMB0GA1UdDgQWBBRbN7ECrPCdVvh58enwy3Dix46h2jAOBgNVHQ8BAf8EBAMC +B4AwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDAzARBglghkgBhvhC +AQEEBAMCBBAwQAYDVR0gBDkwNzA1BgwrBgEEAbIxAQIBAwIwJTAjBggrBgEFBQcC +ARYXaHR0cHM6Ly9zZWN0aWdvLmNvbS9DUFMwQwYDVR0fBDwwOjA4oDagNIYyaHR0 +cDovL2NybC5zZWN0aWdvLmNvbS9TZWN0aWdvUlNBQ29kZVNpZ25pbmdDQS5jcmww +cwYIKwYBBQUHAQEEZzBlMD4GCCsGAQUFBzAChjJodHRwOi8vY3J0LnNlY3RpZ28u +Y29tL1NlY3RpZ29SU0FDb2RlU2lnbmluZ0NBLmNydDAjBggrBgEFBQcwAYYXaHR0 +cDovL29jc3Auc2VjdGlnby5jb20wKwYDVR0RBCQwIoEgam9uYXNAYml0Y29pbmNv +cmVjb2Rlc2lnbmluZy5vcmcwDQYJKoZIhvcNAQELBQADggEBAF/AIXcFBWCC2Red +SHN4Cvko5mdSkDNgzjVFc+OwAJ5RdOgbERde4PnHm3Qmrnx+uMetVnmrC8Fv1Iwb +kkR0bdbWBj6lF6zMsClIN6WJEfY+qfj1qi7wyucu+3OElYRC9bm5Lf0mEHQr8lJ1 +lGvAjPh+/hmxoVNbHFMZ1Ea+BrbjVwiSznt0gzdMh0CispBZKLWCIwRwi+hFjQrw +Z7RLH8HeCJ5Ojl/OTDQqh6AylQ7l9w9KHsUt4Jqy/AnCCyAj2/6xjdwnuo3tCZwb +g/9CydiAacD/83odphEeC2iBa+0wsj9bWmyYKY7S9n0u+wm3wBfZbSVMDDPk/la1 +3qCUDLk= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF4DCCA8igAwIBAgIQLnyHzA6TSlL+lP0ct800rzANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTMwNTA5 -MDAwMDAwWhcNMjgwNTA4MjM1OTU5WjB9MQswCQYDVQQGEwJHQjEbMBkGA1UECBMS -R3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFD -T01PRE8gQ0EgTGltaXRlZDEjMCEGA1UEAxMaQ09NT0RPIFJTQSBDb2RlIFNpZ25p -bmcgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmmJBjd5E0f4rR -3elnMRHrzB79MR2zuWJXP5O8W+OfHiQyESdrvFGRp8+eniWzX4GoGA8dHiAwDvth -e4YJs+P9omidHCydv3Lj5HWg5TUjjsmK7hoMZMfYQqF7tVIDSzqwjiNLS2PgIpQ3 -e9V5kAoUGFEs5v7BEvAcP2FhCoyi3PbDMKrNKBh1SMF5WgjNu4xVjPfUdpA6M0ZQ -c5hc9IVKaw+A3V7Wvf2pL8Al9fl4141fEMJEVTyQPDFGy3CuB6kK46/BAW+QGiPi -XzjbxghdR7ODQfAuADcUuRKqeZJSzYcPe9hiKaR+ML0btYxytEjy4+gh+V5MYnmL -Agaff9ULAgMBAAGjggFRMIIBTTAfBgNVHSMEGDAWgBS7r34CPfqm8TyEjq3uOJjs -2TIy1DAdBgNVHQ4EFgQUKZFg/4pN+uv5pmq4z/nmS71JzhIwDgYDVR0PAQH/BAQD -AgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAwEwYDVR0lBAwwCgYIKwYBBQUHAwMwEQYD -VR0gBAowCDAGBgRVHSAAMEwGA1UdHwRFMEMwQaA/oD2GO2h0dHA6Ly9jcmwuY29t -b2RvY2EuY29tL0NPTU9ET1JTQUNlcnRpZmljYXRpb25BdXRob3JpdHkuY3JsMHEG -CCsGAQUFBwEBBGUwYzA7BggrBgEFBQcwAoYvaHR0cDovL2NydC5jb21vZG9jYS5j -b20vQ09NT0RPUlNBQWRkVHJ1c3RDQS5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9v -Y3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQwFAAOCAgEAAj8COcPu+Mo7id4M -bU2x8U6ST6/COCwEzMVjEasJY6+rotcCP8xvGcM91hoIlP8l2KmIpysQGuCbsQci -GlEcOtTh6Qm/5iR0rx57FjFuI+9UUS1SAuJ1CAVM8bdR4VEAxof2bO4QRHZXavHf -WGshqknUfDdOvf+2dVRAGDZXZxHNTwLk/vPa/HUX2+y392UJI0kfQ1eD6n4gd2HI -TfK7ZU2o94VFB696aSdlkClAi997OlE5jKgfcHmtbUIgos8MbAOMTM1zB5TnWo46 -BLqioXwfy2M6FafUFRunUkcyqfS/ZEfRqh9TTjIwc8Jvt3iCnVz/RrtrIh2IC/gb -qjSm/Iz13X9ljIwxVzHQNuxHoc/Li6jvHBhYxQZ3ykubUa9MCEp6j+KjUuKOjswm -5LLY5TjCqO3GgZw1a6lYYUoKl7RLQrZVnb6Z53BtWfhtKgx/GWBfDJqIbDCsUgmQ -Fhv/K53b0CDKieoofjKOGd97SDMe12X4rsn4gxSTdn1k0I7OvjV9/3IxTZ+evR5s -L6iPDAZQ+4wns3bJ9ObXwzTijIchhmH+v1V04SF3AwpobLvkyanmz1kl63zsRQ55 -ZmjoIs2475iFTZYRPAmK0H+8KCgT+2rKVI2SXM3CZZgGns5IW9S1N5NGQXwH3c/6 -Q++6Z2H/fUnguzB9XIDj5hY5S6c= +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 -MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT -EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR -Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR -6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X -pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC -9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV -/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf -Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z -+pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w -qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah -SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC -u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf -Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq -crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E -FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl -wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM -4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV -2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna -FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ -CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK -boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke -jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL -S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb -QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl -0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB -NVOFBkpdn627G190 +MIIF9TCCA92gAwIBAgIQHaJIMG+bJhjQguCWfTPTajANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgx +MTAyMDAwMDAwWhcNMzAxMjMxMjM1OTU5WjB8MQswCQYDVQQGEwJHQjEbMBkGA1UE +CBMSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRgwFgYDVQQK +Ew9TZWN0aWdvIExpbWl0ZWQxJDAiBgNVBAMTG1NlY3RpZ28gUlNBIENvZGUgU2ln +bmluZyBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIYijTKFehif +SfCWL2MIHi3cfJ8Uz+MmtiVmKUCGVEZ0MWLFEO2yhyemmcuVMMBW9aR1xqkOUGKl +UZEQauBLYq798PgYrKf/7i4zIPoMGYmobHutAMNhodxpZW0fbieW15dRhqb0J+V8 +aouVHltg1X7XFpKcAC9o95ftanK+ODtj3o+/bkxBXRIgCFnoOc2P0tbPBrRXBbZO +oT5Xax+YvMRi1hsLjcdmG0qfnYHEckC14l/vC0X/o84Xpi1VsLewvFRqnbyNVlPG +8Lp5UEks9wO5/i9lNfIi6iwHr0bZ+UYc3Ix8cSjz/qfGFN1VkW6KEQ3fBiSVfQ+n +oXw62oY1YdMCAwEAAaOCAWQwggFgMB8GA1UdIwQYMBaAFFN5v1qqK0rPVIDh2JvA +nfKyA2bLMB0GA1UdDgQWBBQO4TqoUzox1Yq+wbutZxoDha00DjAOBgNVHQ8BAf8E +BAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAUBggrBgEFBQcDAwYI +KwYBBQUHAwgwEQYDVR0gBAowCDAGBgRVHSAAMFAGA1UdHwRJMEcwRaBDoEGGP2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FDZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDB2BggrBgEFBQcBAQRqMGgwPwYIKwYBBQUHMAKGM2h0dHA6 +Ly9jcnQudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FBZGRUcnVzdENBLmNydDAl +BggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG9w0B +AQwFAAOCAgEATWNQ7Uc0SmGk295qKoyb8QAAHh1iezrXMsL2s+Bjs/thAIiaG20Q +BwRPvrjqiXgi6w9G7PNGXkBGiRL0C3danCpBOvzW9Ovn9xWVM8Ohgyi33i/klPeF +M4MtSkBIv5rCT0qxjyT0s4E307dksKYjalloUkJf/wTr4XRleQj1qZPea3FAmZa6 +ePG5yOLDCBaxq2NayBWAbXReSnV+pbjDbLXP30p5h1zHQE1jNfYw08+1Cg4LBH+g +S667o6XQhACTPlNdNKUANWlsvp8gJRANGftQkGG+OY96jk32nw4e/gdREmaDJhlI +lc5KycF/8zoFm/lv34h/wCOe0h5DekUxwZxNqfBZslkZ6GqNKQQCd3xLS81wvjqy +VVp4Pry7bwMQJXcVNIr5NsxDkuS6T/FikyglVyn7URnHoSVAaoRXxrKdsbwcCtp8 +Z359LukoTBh+xHsxQXGaSynsCz1XUNLK3f2eBVHlRHjdAd6xdZgNVCT98E7j4viD +vXK6yz067vBeF5Jobchh+abxKgoLpbn0nu6YMgWFnuv5gynTxix9vTp3Los3QqBq +gu07SqqUEKThDfgXxbZaeTMYkuO1dfih6Y4KJR7kHvGfWocj/5+kUZ77OYARzdu1 +xKeogG/lU9Tg46LC0lsa+jImLWpXcBw8pFguo/NbSwfcMlnzh6cabVg= -----END CERTIFICATE----- diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py deleted file mode 100644 index 66fdf7887f..0000000000 --- a/contrib/zmq/zmq_sub3.4.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -""" - ZMQ example using python3's asyncio - - Bitcoin should be started with the command line arguments: - bitcoind -testnet -daemon \ - -zmqpubrawtx=tcp://127.0.0.1:28332 \ - -zmqpubrawblock=tcp://127.0.0.1:28332 \ - -zmqpubhashtx=tcp://127.0.0.1:28332 \ - -zmqpubhashblock=tcp://127.0.0.1:28332 - - We use the asyncio library here. `self.handle()` installs itself as a - future at the end of the function. Since it never returns with the event - loop having an empty stack of futures, this creates an infinite loop. An - alternative is to wrap the contents of `handle` inside `while True`. - - The `@asyncio.coroutine` decorator and the `yield from` syntax found here - was introduced in python 3.4 and has been deprecated in favor of the `async` - and `await` keywords respectively. - - A blocking example using python 2.7 can be obtained from the git history: - https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py -""" - -import binascii -import asyncio -import zmq -import zmq.asyncio -import signal -import struct -import sys - -if (sys.version_info.major, sys.version_info.minor) < (3, 4): - print("This example only works with Python 3.4 and greater") - sys.exit(1) - -port = 28332 - -class ZMQHandler(): - def __init__(self): - self.loop = asyncio.get_event_loop() - self.zmqContext = zmq.asyncio.Context() - - self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0) - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashtx") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx") - self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % port) - - @asyncio.coroutine - def handle(self) : - msg = yield from self.zmqSubSocket.recv_multipart() - topic = msg[0] - body = msg[1] - sequence = "Unknown" - if len(msg[-1]) == 4: - msgSequence = struct.unpack('<I', msg[-1])[-1] - sequence = str(msgSequence) - if topic == b"hashblock": - print('- HASH BLOCK ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"hashtx": - print('- HASH TX ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"rawblock": - print('- RAW BLOCK HEADER ('+sequence+') -') - print(binascii.hexlify(body[:80])) - elif topic == b"rawtx": - print('- RAW TX ('+sequence+') -') - print(binascii.hexlify(body)) - # schedule ourselves to receive the next message - asyncio.ensure_future(self.handle()) - - def start(self): - self.loop.add_signal_handler(signal.SIGINT, self.stop) - self.loop.create_task(self.handle()) - self.loop.run_forever() - - def stop(self): - self.loop.stop() - self.zmqContext.destroy() - -daemon = ZMQHandler() -daemon.start() diff --git a/depends/Makefile b/depends/Makefile index 50cc77ddeb..dc2a1e626c 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -192,4 +192,6 @@ download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win +$(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) + .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.md b/depends/README.md index 693bc36197..68a83a2aea 100644 --- a/depends/README.md +++ b/depends/README.md @@ -34,7 +34,7 @@ No other options are needed, the paths are automatically configured. #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools #### For Win32/Win64 cross compilation @@ -72,7 +72,7 @@ The following can be set when running make: make FOO=bar NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking - RAPIDCHECK: build rapidcheck (experimental) + RAPIDCHECK: build rapidcheck (experimental, requires cmake) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids diff --git a/depends/config.site.in b/depends/config.site.in index b7a5e795c8..52b9a7eca2 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -67,7 +67,7 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH +PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH if test -n "@AR@"; then AR=@AR@ diff --git a/depends/funcs.mk b/depends/funcs.mk index 15e404e42d..8f03c5f37a 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -76,8 +76,9 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands +# The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) -$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source) +$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= $(1)_build_cmds ?= $(1)_config_cmds ?= @@ -170,15 +171,15 @@ $($(1)_extracted): | $($(1)_fetched) $(AT)mkdir -p $$(@D) $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) $(AT)touch $$@ -$($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted) +$($(1)_preprocessed): | $($(1)_extracted) $(AT)echo Preprocessing $(1)... $(AT)mkdir -p $$(@D) $($(1)_patch_dir) $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) $(AT)touch $$@ -$($(1)_configured): | $($(1)_preprocessed) +$($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) $(AT)echo Configuring $(1)... - $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar xf $($(package)_cached); ) + $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) $(AT)mkdir -p $$(@D) $(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) $(AT)touch $$@ @@ -213,6 +214,14 @@ $(1): | $($(1)_cached_checksum) endef +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + # These functions create the build targets for each package. They must be # broken down into small steps so that each part is done for all packages # before moving on to the next step. Otherwise, a package's info diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk index 5f247e9bf3..c3054cbd1a 100644 --- a/depends/packages/native_biplist.mk +++ b/depends/packages/native_biplist.mk @@ -3,13 +3,13 @@ $(package)_version=1.0.3 $(package)_download_path=https://bitbucket.org/wooster/biplist/downloads $(package)_file_name=biplist-$($(package)_version).tar.gz $(package)_sha256_hash=4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 44d238cc4c..ccd72a99bd 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -22,12 +22,12 @@ define $(package)_extract_cmds echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ - tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ + tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ rm -f toolchain/lib/libc++abi.so* && \ echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ chmod +x toolchain/bin/$(host)-dsymutil && \ - tar --strip-components=1 -xf $($(package)_source) + tar --no-same-owner --strip-components=1 -xf $($(package)_source) endef define $(package)_set_vars diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 116fa25d38..f99b689ecd 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -3,14 +3,14 @@ $(package)_version=1.1.2 $(package)_download_path=https://github.com/al45tair/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=3b3ecb7bf0a5157f5b6010bc3af7c141fb0ad3527084e63336220d22744bc20c -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages $(package)_dependencies=native_biplist define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 306c835656..e60b99dccc 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -3,13 +3,13 @@ $(package)_version=2.0.7 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=6f606d3b6bccd2112aeabf1a063f5b5ece87005a5d7e97c8faca23b916e88838 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_protobuf.mk b/depends/packages/native_protobuf.mk index ce50b366fa..1de8c37d36 100644 --- a/depends/packages/native_protobuf.mk +++ b/depends/packages/native_protobuf.mk @@ -5,7 +5,7 @@ $(package)_file_name=protobuf-$($(package)_version).tar.bz2 $(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910 define $(package)_set_vars -$(package)_config_opts=--disable-shared +$(package)_config_opts=--disable-shared --without-zlib endef define $(package)_config_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index deebf13e98..23cde9ee6d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -125,11 +125,11 @@ define $(package)_extract_cmds echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ - tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ mkdir qttranslations && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ mkdir qttools && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools + tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef define $(package)_preprocess_cmds @@ -138,7 +138,7 @@ define $(package)_preprocess_cmds sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ sed -i.old "s|X11/extensions/XIproto.h|X11/X.h|" qtbase/src/plugins/platforms/xcb/qxcbxsettings.cpp && \ - sed -i.old 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' qtbase/configure && \ + sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \ sed -i.old 's/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, 0)/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, kCGMouseButtonLeft)/' qtbase/src/plugins/platforms/cocoa/qcocoacursor.mm && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk index 19cf1cae2e..a35e091c80 100644 --- a/depends/packages/rapidcheck.mk +++ b/depends/packages/rapidcheck.mk @@ -1,18 +1,17 @@ package=rapidcheck -$(package)_version=10fc0cb -$(package)_download_path=https://github.com/MarcoFalke/rapidcheck/archive -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=9640926223c00af45bce4c7df8b756b5458a89b2ba74cfe3e404467f13ce26df +$(package)_version=3eb9b4ff69f4ff2d9932e8f852c2b2a61d7c20d3 +$(package)_download_path=https://github.com/emil-e/rapidcheck/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=5fbf82755c9a647127e62563be079448ff8b1db9ca80a52a673dd9a88fdb714b define $(package)_config_cmds - cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true . + cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_INSTALL_ALL_EXTRAS=ON endef define $(package)_build_cmds - $(MAKE) && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/include && \ - cp -a include/* $($(package)_staging_dir)$(host_prefix)/include/ && \ - cp -a extras/boost_test/include/rapidcheck/* $($(package)_staging_dir)$(host_prefix)/include/rapidcheck/ && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/lib && \ - cp -a librapidcheck.a $($(package)_staging_dir)$(host_prefix)/lib/ + $(MAKE) rapidcheck +endef + +define $(package)_stage_cmds + $(MAKE) rapidcheck install endef diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 982afd5d56..a0cfe84a3e 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,18 @@ The headless daemon `bitcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Versioning + +The RPC interface might change from one major version of Bitcoin Core to the +next. This makes the RPC interface implicitly versioned on the major version. +The version tuple can be retrieved by e.g. the `getnetworkinfo` RPC in +`version`. + +Usually deprecated features can be re-enabled during the grace-period of one +major version via the `-deprecatedrpc=` command line option. The release notes +of a new major release come with detailed instructions on what RPC features +were deprecated and how to re-enable them temporarily. + ## Security The RPC interface allows other programs to control Bitcoin Core, diff --git a/doc/README.md b/doc/README.md index 8876ffdd72..b4fa933c8e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -57,7 +57,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) -- [Travis CI](travis-ci.md) - [JSON-RPC Interface](JSON-RPC-interface.md) - [Unauthenticated REST Interface](REST-interface.md) - [Shared Libraries](shared-libraries.md) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 02a665008b..c96871ab5f 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -101,6 +101,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 Returns various information about the TX mempool. Only supports JSON as output format. +* loaded : (boolean) if the mempool is fully loaded * size : (numeric) the number of transactions in the TX mempool * bytes : (numeric) size of the TX mempool in bytes * usage : (numeric) total TX mempool memory usage diff --git a/doc/bips.md b/doc/bips.md index 76edc94c29..eb24ce6f66 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.18.0**): * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -15,11 +15,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): * [`BIP 35`](https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki): The 'mempool' protocol message (and the protocol version bump to 60002) has been implemented since **v0.7.0** ([PR #1641](https://github.com/bitcoin/bitcoin/pull/1641)). * [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial Merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). * [`BIP 42`](https://github.com/bitcoin/bips/blob/master/bip-0042.mediawiki): The bug that would have caused the subsidy schedule to resume after block 13440000 was fixed in **v0.9.2** ([PR #3842](https://github.com/bitcoin/bitcoin/pull/3842)). -* [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting *v0.17.0*, whether to send reject messages can be configured with the `-enablebip61` option. +* [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting **v0.17.0**, whether to send reject messages can be configured with the `-enablebip61` option, and support is deprecated as of **v0.18.0**. * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)), and have been activated since *block 419328*. -* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). +* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)) and has been activated since *block 419328*. @@ -33,7 +33,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.17.0**): * [`BIP 145`](https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki): getblocktemplate updates for Segregated Witness as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). -* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): NODE_NETWORK_LIMITED service bit [signaling only] is supported as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)). +* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): The NODE_NETWORK_LIMITED service bit is signalled as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)), and such nodes are connected to as of **v0.17.0** ([PR 10387](https://github.com/bitcoin/bitcoin/pull/10387)). * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). * [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)). diff --git a/doc/build-osx.md b/doc/build-osx.md index 119896dc67..d28a3d97aa 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -82,6 +82,8 @@ Bitcoin Core is now available at `./src/bitcoind` Before running, you may create an empty configuration file: + mkdir -p "/Users/${USER}/Library/Application Support/Bitcoin" + touch "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" diff --git a/doc/build-unix.md b/doc/build-unix.md index 522e3069ce..da65bc347a 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -78,19 +78,13 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in BerkeleyDB is required for the wallet. -**For Ubuntu only:** db4.8 packages are available [here](https://launchpad.net/~bitcoin/+archive/bitcoin). -You can add the repository and install using the following commands: - - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:bitcoin/bitcoin - sudo apt-get update - sudo apt-get install libdb4.8-dev libdb4.8++-dev - Ubuntu and Debian have their own libdb-dev and libdb++-dev packages, but these will install BerkeleyDB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. +Otherwise, you can build from self-compiled `depends` (see above). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) diff --git a/doc/build-windows.md b/doc/build-windows.md index 9641e0d3fd..036c585b44 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -71,6 +71,11 @@ If you want to build the windows installer with `make deploy` you need [NSIS](ht sudo apt install nsis +Acquire the source in the usual way: + + git clone https://github.com/bitcoin/bitcoin.git + cd bitcoin + ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: @@ -87,11 +92,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Once the source code is ready the build steps are below: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -115,11 +116,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Then build using: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends diff --git a/doc/dependencies.md b/doc/dependencies.md index e4be63772c..1b3df62867 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -17,12 +17,12 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L65) | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L64) | -| libsrvg | | | | | | +| librsvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L66) | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | -| Python (tests) | | [3.4](https://www.python.org/downloads) | | | | +| Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | @@ -43,4 +43,4 @@ Some dependencies are not needed in all configurations. The following are some f * ZeroMQ is needed only with the `--with-zmq` option. #### Other -* librsvg is only needed if you need to run `make deploy` on (cross-compliation to) macOS.
\ No newline at end of file +* librsvg is only needed if you need to run `make deploy` on (cross-compilation to) macOS. diff --git a/doc/descriptors.md b/doc/descriptors.md index 5dbcd95e1d..dbdac2c5b6 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -1,11 +1,18 @@ # Support for Output Descriptors in Bitcoin Core -Since Bitcoin Core v0.17, there is support for Output Descriptors in the -`scantxoutset` RPC call. This is a simple language which can be used to -describe collections of output scripts. - -This document describes the language. For the specifics on usage for scanning -the UTXO set, see the `scantxoutset` RPC help. +Since Bitcoin Core v0.17, there is support for Output Descriptors. This is a +simple language which can be used to describe collections of output scripts. +Supporting RPCs are: +- `scantxoutset` takes as input descriptors to scan for, and also reports + specialized descriptors for the matching UTXOs. +- `getdescriptorinfo` analyzes a descriptor, and reports a canonicalized version + with checksum added. +- `deriveaddresses` takes as input a descriptor and computes the corresponding + addresses. +- `listunspent` outputs a specialized descriptor for the reported unspent outputs. + +This document describes the language. For the specifics on usage, see the RPC +documentation for the functions mentioned above. ## Features diff --git a/doc/developer-notes.md b/doc/developer-notes.md index f765346cd8..cf071167c4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -144,7 +144,7 @@ For example, to describe a function use: */ bool function(int arg1, const char *arg2) ``` -A complete list of `@xxx` commands can be found at http://www.stack.nl/~dimitri/doxygen/manual/commands.html. +A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. @@ -185,7 +185,7 @@ Not OK (used plenty in the current source, but not picked up): // ``` -A full list of comment syntaxes picked up by Doxygen can be found at https://www.stack.nl/~dimitri/doxygen/manual/docblocks.html, +A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. @@ -620,8 +620,8 @@ class AddressBookPage Mode m_mode; } -AddressBookPage::AddressBookPage(Mode _mode) : - m_mode(_mode) +AddressBookPage::AddressBookPage(Mode _mode) + : m_mode(_mode) ... ``` @@ -952,8 +952,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: from there. - A RPC method must either be a wallet method or a non-wallet method. Do not - introduce new methods such as `signrawtransaction` that differ in behavior - based on presence of a wallet. + introduce new methods that differ in behavior based on presence of a wallet. - *Rationale*: as well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be diff --git a/doc/productivity.md b/doc/productivity.md index 862017290d..a93228ebdb 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -8,6 +8,7 @@ Table of Contents * [Cache compilations with `ccache`](#cache-compilations-with-ccache) * [Disable features with `./configure`](#disable-features-with-configure) * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) + * [Only build what you need](#only-build-what-you-need) * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Writing code](#writing-code) * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) @@ -32,6 +33,17 @@ Install `ccache` through your distribution's package manager, and run `./configu To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up. +To get the most out of ccache, put something like this in `~/.ccache/ccache.conf`: + +``` +max_size = 50.0G # or whatever cache size you prefer; default is 5G; 0 means unlimited +base_dir = /home/yourname # or wherever you keep your source files +``` + +Note: base_dir is required for ccache to share cached compiles of the same file across different repositories / paths; it will only do this for paths under base_dir. So this option is required for effective use of ccache with git worktrees (described below). + +You _must not_ set base_dir to "/", or anywhere that contains system headers (according to the ccache docs). + ### Disable features with `./configure` After running `./autogen.sh`, which generates the `./configure` file, use `./configure --help` to identify features that you can disable to save on compilation time. A few common flags: @@ -43,6 +55,8 @@ After running `./autogen.sh`, which generates the `./configure` file, use `./con --without-gui ``` +If you do need the wallet enabled, it is common for devs to add `--with-incompatible-bdb`. This uses your system bdb version for the wallet, so you don't have to find a copy of bdb 4.8. Wallets from such a build will be incompatible with any release binary (and vice versa), so use with caution on mainnet. + ### Make use of your threads with `make -j` If you have multiple threads on your machine, you can tell `make` to utilize all of them with: @@ -51,6 +65,20 @@ If you have multiple threads on your machine, you can tell `make` to utilize all make -j"$(($(nproc)+1))" ``` +### Only build what you need + +When rebuilding during development, note that running `make`, without giving a target, will do a lot of work you probably don't need. It will build the GUI (unless you've disabled it) and all the tests (which take much longer to build than the app does). + +Obviously, it is important to build and run the tests at appropriate times -- but when you just want a quick compile to check your work, consider picking one or a set of build targets relevant to what you're working on, e.g.: + +```sh +make src/bitcoind src/bitcoin-cli +make src/qt/bitcoin-qt +make -C src bitcoin_bench +``` + +(You can and should combine this with `-j`, as above, for a parallel build.) + ### Multiple working directories with `git worktrees` If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. @@ -76,7 +104,7 @@ Writing code ### Format C/C++/Protobuf diffs with `clang-format-diff.py` -See [contrib/devtools/README.md](contrib/devtools/README.md#clang-format-diff.py). +See [contrib/devtools/README.md](/contrib/devtools/README.md#clang-format-diff.py). ### Format Python diffs with `yapf-diff.py` @@ -136,7 +164,7 @@ This will add an `upstream-pull` remote to your git repository, which can be fet ### Diff the diffs with `git range-diff` -It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). `git range-diff` can help solve this problem by diffing the diffs. +It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). [git range-diff](https://git-scm.com/docs/git-range-diff) (Git >= 2.19) can help solve this problem by diffing the diffs. For example, to identify the differences between your previously reviewed diffs P1-5, and the new diffs P1-2,N3-4 as illustrated below: ``` @@ -152,7 +180,19 @@ You can do: git range-diff master previously-reviewed-head new-head ``` -Note that `git range-diff` also work for rebases. +Note that `git range-diff` also work for rebases: + +``` + P1--P2--P3--P4--P5 <-- previously-reviewed-head + / +...--m--m1--m2--m3 <-- master + \ + P1--P2--N3--N4 <-- new-head (with P3 modified, P4 & P5 squashed) + +PREV=P5 N=4 && git range-diff `git merge-base --all HEAD $PREV`...$PREV HEAD~$N...HEAD +``` + +Where `P5` is the commit you last reviewed and `4` is the number of commits in the new version. ----- diff --git a/doc/psbt.md b/doc/psbt.md index 560b45ef31..9d85af0348 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -67,6 +67,9 @@ hardware implementations will typically implement multiple roles simultaneously. input a PSBT, adds UTXO, key, and script data to inputs and outputs that miss it, and optionally signs inputs. Where possible it also finalizes the partial signatures. +- **`utxoupdatepsbt` (Updater)** is a node RPC that takes a PSBT and updates it + to include information available from the UTXO set (works only for SegWit + inputs). - **`finalizepsbt` (Finalizer, Extractor)** is a utility RPC that finalizes any partial signatures, and if all inputs are finalized, converts the result to a fully signed transaction which can be broadcast with `sendrawtransaction`. @@ -74,8 +77,15 @@ hardware implementations will typically implement multiple roles simultaneously. can be used at any point in the workflow to merge information added to different versions of the same PSBT. In particular it is useful to combine the output of multiple Updaters or Signers. +- **`joinpsbts`** (Creator) is a utility RPC that joins multiple PSBTs together, + concatenating the inputs and outputs. This can be used to construct CoinJoin + transactions. - **`decodepsbt`** is a diagnostic utility RPC which will show all information in a PSBT in human-readable form, as well as compute its eventual fee if known. +- **`analyzepsbt`** is a utility RPC that examines an RPC and reports the + next steps in the workflow if known, computes the fee of the resulting + transaction, and estimates the weight and feerate if possible. + ### Workflows diff --git a/doc/release-notes-14021.md b/doc/release-notes-14021.md deleted file mode 100644 index 4797a95bdb..0000000000 --- a/doc/release-notes-14021.md +++ /dev/null @@ -1,11 +0,0 @@ -Miscellaneous RPC Changes -------------------------- -- Descriptors with key origin information imported through `importmulti` will have their key origin information stored in the wallet for use with creating PSBTs. -- If `bip32derivs` of both `walletprocesspsbt` and `walletcreatefundedpsbt` is set to true but the key metadata for a public key has not been updated yet, then that key will have a derivation path as if it were just an independent key (i.e. no derivation path and its master fingerprint is itself) - -Miscellaneous Wallet changes ----------------------------- - -- The key metadata will need to be upgraded the first time that the HD seed is available. -For unencrypted wallets this will occur on wallet loading. -For encrypted wallets this will occur the first time the wallet is unlocked. diff --git a/doc/release-notes-14054.md b/doc/release-notes-14054.md new file mode 100644 index 0000000000..d8cad369c5 --- /dev/null +++ b/doc/release-notes-14054.md @@ -0,0 +1,7 @@ +P2P changes +----------- + +BIP 61 reject messages were deprecated in v0.18. They are now disabled by +default, but can be enabled by setting the `-enablebip61` command line option. +BIP 61 reject messages will be removed entirely in a future version of +Bitcoin Core. diff --git a/doc/release-notes-14481.md b/doc/release-notes-14481.md deleted file mode 100644 index ea8fc3c34e..0000000000 --- a/doc/release-notes-14481.md +++ /dev/null @@ -1,9 +0,0 @@ -Low-level RPC changes ----------------------- - -The `listunspent` RPC has been modified so that it also returns `witnessScript`, -the witness script in the case of a P2WSH or P2SH-P2WSH output. - -The `signrawtransactionwithkey` and `signrawtransactionwithwallet` RPCs have been -modified so that they also optionally accept a `witnessScript`, the witness script in the -case of a P2WSH or P2SH-P2WSH output. This is compatible with the change to `listunspent`. diff --git a/doc/release-notes-14491.md b/doc/release-notes-14491.md deleted file mode 100644 index 1cf36e85cf..0000000000 --- a/doc/release-notes-14491.md +++ /dev/null @@ -1,5 +0,0 @@ -Descriptor import support ---------------------- - -The `importmulti` RPC now supports importing of addresses from descriptors. A "desc" parameter can be provided instead of the "scriptPubKey" in a request, as well as an optional range for ranged descriptors to specify the start and end of the range to import. More information about -descriptors can be found [here](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). diff --git a/doc/release-notes-14667.md b/doc/release-notes-14667.md deleted file mode 100644 index 5cb1d0aee7..0000000000 --- a/doc/release-notes-14667.md +++ /dev/null @@ -1,4 +0,0 @@ -New RPC methods ------------- - -- `deriveaddresses` returns one or more addresses corresponding to an [output descriptor](/doc/descriptors.md). diff --git a/doc/release-notes-15226.md b/doc/release-notes-15226.md deleted file mode 100644 index 3be84db3e9..0000000000 --- a/doc/release-notes-15226.md +++ /dev/null @@ -1,8 +0,0 @@ -Miscellaneous RPC changes ------------- - -- The RPC `createwallet` now has an optional `blank` argument that can be used to create a blank wallet. -Blank wallets do not have any keys or HD seed. -They cannot be opened in software older than 0.18. -Once a blank wallet has a HD seed set (by using `sethdseed`) or private keys, scripts, addresses, and other watch only things have been imported, the wallet is no longer blank and can be opened in 0.17.x. -Encrypting a blank wallet will also set a HD seed for it. diff --git a/doc/release-notes-15393.md b/doc/release-notes-15393.md deleted file mode 100644 index f478dc798d..0000000000 --- a/doc/release-notes-15393.md +++ /dev/null @@ -1,4 +0,0 @@ -Dependencies ------------- - -- The minimum required version of QT has been increased from 5.2 to 5.5.1 (the [depends system](https://github.com/bitcoin/bitcoin/blob/master/depends/README.md) provides 5.9.7) diff --git a/doc/release-notes-15566.md b/doc/release-notes-15566.md new file mode 100644 index 0000000000..49964d7550 --- /dev/null +++ b/doc/release-notes-15566.md @@ -0,0 +1,3 @@ +Miscellaneous CLI Changes +------------------------- +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to `chain` and now returns the current network name as defined in BIP70 (main, test, regtest).
\ No newline at end of file diff --git a/doc/release-notes-15620.md b/doc/release-notes-15620.md new file mode 100644 index 0000000000..bf89a70a4e --- /dev/null +++ b/doc/release-notes-15620.md @@ -0,0 +1,13 @@ +Updated RPCs +------------ + +* The -maxtxfee setting no longer has any effect on non-wallet RPCs. + + The `sendrawtransaction` and `testmempoolaccept` RPC methods previously + accepted an `allowhighfees` parameter to fail the mempool acceptance in case + the transaction's fee would exceed the value of the command line argument + `-maxtxfee`. To uncouple the RPCs from the global option, they now have a + hardcoded default for the maximum transaction fee, that can be changed for + both RPCs on a per-call basis with the `maxfeerate` parameter. The + `allowhighfees` boolean option has been removed and replaced by the + `maxfeerate` numeric option. diff --git a/doc/release-notes-15637.md b/doc/release-notes-15637.md new file mode 100644 index 0000000000..048d5e7218 --- /dev/null +++ b/doc/release-notes-15637.md @@ -0,0 +1,3 @@ +RPC changes +----------- +In getmempoolancestors, getmempooldescendants, getmempoolentry and getrawmempool RPCs, to be consistent with the returned value and other RPCs such as getrawtransaction, vsize has been added and size is now deprecated. size will only be returned if bitcoind is started with `-deprecatedrpc=size`.
\ No newline at end of file diff --git a/doc/release-notes-15849.md b/doc/release-notes-15849.md new file mode 100644 index 0000000000..a1df31f250 --- /dev/null +++ b/doc/release-notes-15849.md @@ -0,0 +1,6 @@ +Thread names in logs +-------------------- + +On platforms supporting `thread_local`, log lines can be prefixed with the name +of the thread that caused the log. To enable this behavior, use +`-logthreadnames=1`. diff --git a/doc/release-notes.md b/doc/release-notes.md index a6408cf1e6..0de0f563b1 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -24,14 +24,9 @@ shut down (which might take a few minutes for older versions), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). -The first time you run version 0.15.0, your chainstate database will be converted to a -new format, which will take anywhere from a few minutes to half an hour, -depending on the speed of your machine. - -Note that the block database format also changed in version 0.8.0 and there is no -automatic upgrade code from before version 0.8 to version 0.15.0. Upgrading -directly from 0.7.x and earlier without redownloading the blockchain is not supported. -However, as usual, old wallet versions are still supported. +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. Downgrading warning ------------------- @@ -66,313 +61,32 @@ platform. Notable changes =============== -Mining ------- - -- Calls to `getblocktemplate` will fail if the segwit rule is not specified. - Calling `getblocktemplate` without segwit specified is almost certainly - a misconfiguration since doing so results in lower rewards for the miner. - Failed calls will produce an error message describing how to enable the - segwit rule. - -Configuration option changes ----------------------------- - -- A warning is printed if an unrecognized section name is used in the - configuration file. Recognized sections are `[test]`, `[main]`, and - `[regtest]`. - -- Four new options are available for configuring the maximum number of - messages that ZMQ will queue in memory (the "high water mark") before - dropping additional messages. The default value is 1,000, the same as - was used for previous releases. See the [ZMQ - documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/zmq.md#usage) - for details. - -- The `enablebip61` option (introduced in Bitcoin Core 0.17.0) is - used to toggle sending of BIP 61 reject messages. Reject messages have no use - case on the P2P network and are only logged for debugging by most network - nodes. The option will now by default be off for improved privacy and security - as well as reduced upload usage. The option can explicitly be turned on for - local-network debugging purposes. - -- The `rpcallowip` option can no longer be used to automatically listen - on all network interfaces. Instead, the `rpcbind` parameter must also - be used to specify the IP addresses to listen on. Listening for RPC - commands over a public network connection is insecure and should be - disabled, so a warning is now printed if a user selects such a - configuration. If you need to expose RPC in order to use a tool - like Docker, ensure you only bind RPC to your localhost, e.g. `docker - run [...] -p 127.0.0.1:8332:8332` (this is an extra `:8332` over the - normal Docker port specification). - -- The `rpcpassword` option now causes a startup error if the password - set in the configuration file contains a hash character (#), as it's - ambiguous whether the hash character is meant for the password or as a - comment. - -- The `whitelistforcerelay` option is used to relay transactions from - whitelisted peers even when not accepted to the mempool. This option now - defaults to being off, so that changes in policy and disconnect/ban behavior - will not cause a node that is whitelisting another to be dropped by peers. - Users can still explicitly enable this behavior with the command line option - (and may want to consider [contacting](https://bitcoincore.org/en/contact/) - the Bitcoin Core project to let us know about their - use-case, as this feature could be deprecated in the future). - -Documentation -------------- - -- A new short - [document](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) - about the JSON-RPC interface describes cases where the results of an - RPC might contain inconsistencies between data sourced from different - subsystems, such as wallet state and mempool state. A note is added - to the [REST interface documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) - indicating that the same rules apply. - -- Further information is added to the [JSON-RPC - documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) - about how to secure this interface. - -- A new [document](https://github.com/bitcoin/bitcoin/blob/master/doc/bitcoin-conf.md) - about the `bitcoin.conf` file describes how to use it to configure - Bitcoin Core. - -- A new document introduces Bitcoin Core's BIP174 - [Partially-Signed Bitcoin Transactions (PSBT)](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) - interface, which is used to allow multiple programs to collaboratively - work to create, sign, and broadcast new transactions. This is useful - for offline (cold storage) wallets, multisig wallets, coinjoin - implementations, and many other cases where two or more programs need - to interact to generate a complete transaction. - -- The [output script descriptor](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md) - documentation has been updated with information about new features in - this still-developing language for describing the output scripts that - a wallet or other program wants to receive notifications for, such as - which addresses it wants to know received payments. The language is - currently used in the `scantxoutset` RPC and is expected to be adapted - to other RPCs and to the underlying wallet structure. - -Build system changes --------------------- - -- A new `--disable-bip70` option may be passed to `./configure` to - prevent Bitcoin-Qt from being built with support for the BIP70 payment - protocol or from linking libssl. As the payment protocol has exposed - Bitcoin Core to libssl vulnerabilities in the past, builders who don't - need BIP70 support are encouraged to use this option to reduce their - exposure to future vulnerabilities. - -Deprecated or removed RPCs --------------------------- - -- The `signrawtransaction` RPC is removed after being deprecated and - hidden behind a special configuration option in version 0.17.0. - -- The 'account' API is removed after being deprecated in v0.17. The - 'label' API was introduced in v0.17 as a replacement for accounts. - See the [release notes from v0.17](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.17.0.md#label-and-account-apis-for-wallet) - for a full description of the changes from the 'account' API to the - 'label' API. - -- The `addwitnessaddress` RPC is removed after being deprecated in - version 0.13.0. - -- The wallet's `generate` RPC method is deprecated and will be fully - removed in a subsequent major version. This RPC is only used for - testing, but its implementation reached across multiple subsystems - (wallet and mining), so it is being deprecated to simplify the - wallet-node interface. Projects that are using `generate` for testing - purposes should transition to using the `generatetoaddress` RPC, which - does not require or use the wallet component. Calling - `generatetoaddress` with an address returned by the `getnewaddress` - RPC gives the same functionality as the old `generate` RPC. To - continue using `generate` in this version, restart bitcoind with the - `-deprecatedrpc=generate` configuration option. - -New RPCs --------- - -- The `getnodeaddresses` RPC returns peer addresses known to this - node. It may be used to find nodes to connect to without using a DNS - seeder. - -- The `listwalletdir` RPC returns a list of wallets in the wallet - directory (either the default wallet directory or the directory - configured by the `-walletdir` parameter). - -- The `getrpcinfo` returns runtime details of the RPC server. At the - moment, it returns an array of the currently active commands and how - long they've been running. - Updated RPCs ------------ -Note: some low-level RPC changes mainly useful for testing are described -in the Low-level Changes section below. - -- The `getpeerinfo` RPC now returns an additional `minfeefilter` field - set to the peer's BIP133 fee filter. You can use this to detect that - you have peers that are willing to accept transactions below the - default minimum relay fee. - -- The mempool RPCs, such as `getrawmempool` with `verbose=true`, now - return an additional "bip125-replaceable" value indicating whether the - transaction (or its unconfirmed ancestors) opts-in to asking nodes and - miners to replace it with a higher-feerate transaction spending any of - the same inputs. - -- The `settxfee` RPC previously silently ignored attempts to set the fee - below the allowed minimums. It now prints a warning. The special - value of "0" may still be used to request the minimum value. - -- The `getaddressinfo` RPC now provides an `ischange` field indicating - whether the wallet used the address in a change output. - -- The `importmulti` RPC has been updated to support P2WSH, P2WPKH, - P2SH-P2WPKH, and P2SH-P2WSH. Requests for P2WSH and P2SH-P2WSH accept - an additional `witnessscript` parameter. - -- The `importmulti` RPC now returns an additional `warnings` field for - each request with an array of strings explaining when fields are being - ignored or are inconsistent, if there are any. - -- The `getaddressinfo` RPC now returns an additional `solvable` boolean - field when Bitcoin Core knows enough about the address's scriptPubKey, - optional redeemScript, and optional witnessScript in order for the - wallet to be able to generate an unsigned input spending funds sent to - that address. - -- The `getaddressinfo`, `listunspent`, and `scantxoutset` RPCs now - return an additional `desc` field that contains an output descriptor - containing all key paths and signing information for the address - (except for the private key). The `desc` field is only returned for - `getaddressinfo` and `listunspent` when the address is solvable. - -- The `importprivkey` RPC will preserve previously-set labels for - addresses or public keys corresponding to the private key being - imported. For example, if you imported a watch-only address with the - label "cold wallet" in earlier releases of Bitcoin Core, subsequently - importing the private key would default to resetting the address's - label to the default empty-string label (""). In this release, the - previous label of "cold wallet" will be retained. If you optionally - specify any label besides the default when calling `importprivkey`, - the new label will be applied to the address. - -- See the [Mining](#mining) section for changes to `getblocktemplate`. - -- The `getmininginfo` RPC now omits `currentblockweight` and `currentblocktx` - when a block was never assembled via RPC on this node. - -- The `getrawtransaction` RPC & REST endpoints no longer check the - unspent UTXO set for a transaction. The remaining behaviors are as - follows: 1. If a blockhash is provided, check the corresponding block. - 2. If no blockhash is provided, check the mempool. 3. If no blockhash - is provided but txindex is enabled, also check txindex. - -- The `unloadwallet` RPC is now synchronous, meaning it will not return - until the wallet is fully unloaded. - -REST changes ------------- - -- A new `/rest/blockhashbyheight/` endpoint is added for fetching the - hash of the block in the current best blockchain based on its height - (how many blocks it is after the Genesis Block). - -Graphical User Interface (GUI) ------------------------------- +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. -- A new Window menu is added alongside the existing File, Settings, and - Help menus. Several items from the other menus that opened new - windows have been moved to this new Window menu. +* The `sendmany` RPC had an argument `minconf` that was not well specified and + would lead to RPC errors even when the wallet's coin selection would succeed. + The `sendtoaddress` RPC never had this check, so to normalize the behavior, + `minconf` is now ignored in `sendmany`. If the coin selection does not + succeed due to missing coins, it will still throw an RPC error. Be reminded + that coin selection is influenced by the `-spendzeroconfchange`, + `-limitancestorcount`, `-limitdescendantcount` and `-walletrejectlongchains` + command line arguments. -- In the Send tab, the checkbox for "pay only the required fee" - has been removed. Instead, the user can simply decrease the value in - the Custom Feerate field all the way down to the node's configured - minimum relay fee. - -- In the Overview tab, the watch-only balance will be the only - balance shown if the wallet was created using the `createwallet` RPC - and the `disable_private_keys` parameter was set to true. - -- The launch-on-startup option is no longer available on macOS if - compiled with macosx min version greater than 10.11 (use - CXXFLAGS="-mmacosx-version-min=10.11" - CFLAGS="-mmacosx-version-min=10.11" for setting the deployment - sdk version) - -Tools ----- - -- A new `bitcoin-wallet` tool is now distributed alongside Bitcoin - Core's other executables. Without needing to use any RPCs, this tool - can currently create a new wallet file or display some basic - information about an existing wallet, such as whether the wallet is - encrypted, whether it uses an HD seed, how many transactions it - contains, and how many address book entries it has. Low-level changes ================= -RPC ---- - -- The `submitblock` RPC previously returned the reason a rejected block - was invalid the first time it processed that block but returned a - generic "duplicate" rejection message on subsequent occasions it - processed the same block. It now always returns the fundamental - reason for rejecting an invalid block and only returns "duplicate" for - valid blocks it has already accepted. - -- A new `submitheader` RPC allows submitting block headers independently - from their block. This is likely only useful for testing. - Configuration -------------- - -- The `-usehd` configuration option was removed in version 0.16. From - that version onwards, all new wallets created are hierarchical - deterministic wallets. This release makes specifying `-usehd` an - invalid configuration option. - -Network -------- - -- This release allows peers that your node automatically disconnected - for misbehavior (e.g. sending invalid data) to reconnect to your node - if you have unused incoming connection slots. If your slots fill up, - a misbehaving node will be disconnected to make room for nodes without - a history of problems (unless the misbehaving node helps your node in - some other way, such as by connecting to a part of the Internet from - which you don't have many other peers). Previously, Bitcoin Core - banned the IP addresses of misbehaving peers for a period of time - (default of 1 day); this was easily circumvented by attackers with - multiple IP addresses. If you manually ban a peer, such as by using - the `setban` RPC, all connections from that peer will still be - rejected. - -Security --------- - -- This release changes the Random Number Generator (RNG) used from - OpenSSL to Bitcoin Core's own implementation, although entropy - gathered by Bitcoin Core is fed out to OpenSSL and then read back in - when the program needs strong randomness. This moves Bitcoin Core a - little closer to no longer needing to depend on OpenSSL, a dependency - that has caused security issues in the past. - -Changes for particular platforms --------------------------------- +------------ -- On macOS, Bitcoin Core now opts out of application CPU throttling - ("app nap") during initial blockchain download, when catching up from - over 100 blocks behind the current chain tip, or when reindexing chain - data. This helps prevent these operations from taking an excessively - long time because the operating system is attempting to conserve - power. +* An error is issued where previously a warning was issued when a setting in + the config file was specified in the default section, but not overridden for + the selected network. This change takes only effect if the selected network + is not mainnet. Credits ======= @@ -380,4 +94,4 @@ Credits Thanks to everyone who directly contributed to this release: -As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.18.0.md b/doc/release-notes/release-notes-0.18.0.md new file mode 100644 index 0000000000..3ca7d52243 --- /dev/null +++ b/doc/release-notes/release-notes-0.18.0.md @@ -0,0 +1,1224 @@ +Bitcoin Core version 0.18.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.18.0/> + +This is a new major version release, including new features, various bug +fixes and performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has +completely shut down (which might take a few minutes for older +versions), then run the installer (on Windows) or just copy over +`/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on +Linux). + +The first time you run version 0.15.0 or newer, your chainstate database +will be converted to a new format, which will take anywhere from a few +minutes to half an hour, depending on the speed of your machine. + +Note that the block database format also changed in version 0.8.0 and +there is no automatic upgrade code from before version 0.8 to version +0.15.0 or later. Upgrading directly from 0.7.x and earlier without +redownloading the blockchain is not supported. However, as usual, old +wallet versions are still supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not +recommended to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +built using Qt 5.9.x, which doesn't support versions of macOS older than +10.10. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +In addition to previously-supported CPU platforms, this release's +pre-compiled distribution also provides binaries for the RISC-V +platform. + +If you are using the `systemd` unit configuration file located at +`contrib/init/bitcoind.service`, it has been changed to use +`/var/lib/bitcoind` as the data directory instead of +`~bitcoin/.bitcoin`. When switching over to the new configuration file, +please make sure that the filesystem on which `/var/lib/bitcoind` will +exist has enough space (check using `df -h /var/lib/bitcoind`), and +optionally copy over your existing data directory. See the [systemd init +file section](#systemd-init-file) for more details. + +Known issues +============ + +Wallet GUI +---------- + +For advanced users who have both (1) enabled coin control features, and +(2) are using multiple wallets loaded at the same time: The coin control +input selection dialog can erroneously retain wrong-wallet state when +switching wallets using the dropdown menu. For now, it is recommended +not to use coin control features with multiple wallets loaded. + +Notable changes +=============== + +Mining +------ + +- Calls to `getblocktemplate` will fail if the segwit rule is not + specified. Calling `getblocktemplate` without segwit specified is + almost certainly a misconfiguration since doing so results in lower + rewards for the miner. Failed calls will produce an error message + describing how to enable the segwit rule. + +Configuration option changes +---------------------------- + +- A warning is printed if an unrecognized section name is used in the + configuration file. Recognized sections are `[test]`, `[main]`, and + `[regtest]`. + +- Four new options are available for configuring the maximum number of + messages that ZMQ will queue in memory (the "high water mark") before + dropping additional messages. The default value is 1,000, the same as + was used for previous releases. See the [ZMQ + documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/zmq.md#usage) + for details. + +- The `rpcallowip` option can no longer be used to automatically listen + on all network interfaces. Instead, the `rpcbind` parameter must be + used to specify the IP addresses to listen on. Listening for RPC + commands over a public network connection is insecure and should be + disabled, so a warning is now printed if a user selects such a + configuration. If you need to expose RPC in order to use a tool like + Docker, ensure you only bind RPC to your localhost, e.g. `docker run + [...] -p 127.0.0.1:8332:8332` (this is an extra `:8332` over the + normal Docker port specification). + +- The `rpcpassword` option now causes a startup error if the password + set in the configuration file contains a hash character (#), as it's + ambiguous whether the hash character is meant for the password or as a + comment. + +- The `whitelistforcerelay` option is used to relay transactions from + whitelisted peers even when not accepted to the mempool. This option + now defaults to being off, so that changes in policy and + disconnect/ban behavior will not cause a node that is whitelisting + another to be dropped by peers. Users can still explicitly enable + this behavior with the command line option (and may want to consider + [contacting](https://bitcoincore.org/en/contact/) the Bitcoin Core + project to let us know about their use-case, as this feature could be + deprecated in the future). + +systemd init file +----------------- + +The systemd init file (`contrib/init/bitcoind.service`) has been changed +to use `/var/lib/bitcoind` as the data directory instead of +`~bitcoin/.bitcoin`. This change makes Bitcoin Core more consistent with +other services, and makes the systemd init config more consistent with +existing Upstart and OpenRC configs. + +The configuration, PID, and data directories are now completely managed +by systemd, which will take care of their creation, permissions, etc. +See [`systemd.exec(5)`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=) +for more details. + +When using the provided init files under `contrib/init`, overriding the +`datadir` option in `/etc/bitcoin/bitcoin.conf` will have no effect. +This is because the command line arguments specified in the init files +take precedence over the options specified in +`/etc/bitcoin/bitcoin.conf`. + + +Documentation +------------- + +- A new short [document](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) + about the JSON-RPC interface describes cases where the results of an + RPC might contain inconsistencies between data sourced from different + subsystems, such as wallet state and mempool state. A note is added + to the [REST interface documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) + indicating that the same rules apply. + +- Further information is added to the [JSON-RPC + documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) + about how to secure this interface. + +- A new [document](https://github.com/bitcoin/bitcoin/blob/master/doc/bitcoin-conf.md) + about the `bitcoin.conf` file describes how to use it to configure + Bitcoin Core. + +- A new document introduces Bitcoin Core's BIP174 [Partially-Signed + Bitcoin Transactions + (PSBT)](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) + interface, which is used to allow multiple programs to collaboratively + work to create, sign, and broadcast new transactions. This is useful + for offline (cold storage) wallets, multisig wallets, coinjoin + implementations, and many other cases where two or more programs need + to interact to generate a complete transaction. + +- The [output script + descriptor](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md) + documentation has been updated with information about new features in + this still-developing language for describing the output scripts that + a wallet or other program wants to receive notifications for, such as + which addresses it wants to know received payments. The language is + currently used in multiple new and updated RPCs described in these + release notes and is expected to be adapted to other RPCs and to the + underlying wallet structure. + +Build system changes +-------------------- + +- A new `--disable-bip70` option may be passed to `./configure` to + prevent Bitcoin-Qt from being built with support for the BIP70 payment + protocol or from linking libssl. As the payment protocol has exposed + Bitcoin Core to libssl vulnerabilities in the past, builders who don't + need BIP70 support are encouraged to use this option to reduce their + exposure to future vulnerabilities. + +- The minimum required version of Qt (when building the GUI) has been + increased from 5.2 to 5.5.1 (the [depends + system](https://github.com/bitcoin/bitcoin/blob/master/depends/README.md) + provides 5.9.7) + +New RPCs +-------- + +- `getnodeaddresses` returns peer addresses known to this node. It may + be used to find nodes to connect to without using a DNS seeder. + +- `listwalletdir` returns a list of wallets in the wallet directory + (either the default wallet directory or the directory configured by + the `-walletdir` parameter). + +- `getrpcinfo` returns runtime details of the RPC server. At the moment, + it returns an array of the currently active commands and how long + they've been running. + +- `deriveaddresses` returns one or more addresses corresponding to an + [output descriptor](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). + +- `getdescriptorinfo` accepts a descriptor and returns information about + it, including its computed checksum. + +- `joinpsbts` merges multiple distinct PSBTs into a single PSBT. The + multiple PSBTs must have different inputs. The resulting PSBT will + contain every input and output from all of the PSBTs. Any signatures + provided in any of the PSBTs will be dropped. + +- `analyzepsbt` examines a PSBT and provides information about what + the PSBT contains and the next steps that need to be taken in order + to complete the transaction. For each input of a PSBT, `analyzepsbt` + provides information about what information is missing for that + input, including whether a UTXO needs to be provided, what pubkeys + still need to be provided, which scripts need to be provided, and + what signatures are still needed. Every input will also list which + role is needed to complete that input, and `analyzepsbt` will also + list the next role in general needed to complete the PSBT. + `analyzepsbt` will also provide the estimated fee rate and estimated + virtual size of the completed transaction if it has enough + information to do so. + +- `utxoupdatepsbt` searches the set of Unspent Transaction Outputs + (UTXOs) to find the outputs being spent by the partial transaction. + PSBTs need to have the UTXOs being spent to be provided because + the signing algorithm requires information from the UTXO being spent. + For segwit inputs, only the UTXO itself is necessary. For + non-segwit outputs, the entire previous transaction is needed so + that signers can be sure that they are signing the correct thing. + Unfortunately, because the UTXO set only contains UTXOs and not full + transactions, `utxoupdatepsbt` will only add the UTXO for segwit + inputs. + +Updated RPCs +------------ + +Note: some low-level RPC changes mainly useful for testing are described +in the Low-level Changes section below. + +- `getpeerinfo` now returns an additional `minfeefilter` field set to + the peer's BIP133 fee filter. You can use this to detect that you + have peers that are willing to accept transactions below the default + minimum relay fee. + +- The mempool RPCs, such as `getrawmempool` with `verbose=true`, now + return an additional "bip125-replaceable" value indicating whether the + transaction (or its unconfirmed ancestors) opts-in to asking nodes and + miners to replace it with a higher-feerate transaction spending any of + the same inputs. + +- `settxfee` previously silently ignored attempts to set the fee below + the allowed minimums. It now prints a warning. The special value of + "0" may still be used to request the minimum value. + +- `getaddressinfo` now provides an `ischange` field indicating whether + the wallet used the address in a change output. + +- `importmulti` has been updated to support P2WSH, P2WPKH, P2SH-P2WPKH, + and P2SH-P2WSH. Requests for P2WSH and P2SH-P2WSH accept an additional + `witnessscript` parameter. + +- `importmulti` now returns an additional `warnings` field for each + request with an array of strings explaining when fields are being + ignored or are inconsistent, if there are any. + +- `getaddressinfo` now returns an additional `solvable` boolean field + when Bitcoin Core knows enough about the address's scriptPubKey, + optional redeemScript, and optional witnessScript in order for the + wallet to be able to generate an unsigned input spending funds sent to + that address. + +- The `getaddressinfo`, `listunspent`, and `scantxoutset` RPCs now + return an additional `desc` field that contains an output descriptor + containing all key paths and signing information for the address + (except for the private key). The `desc` field is only returned for + `getaddressinfo` and `listunspent` when the address is solvable. + +- `importprivkey` will preserve previously-set labels for addresses or + public keys corresponding to the private key being imported. For + example, if you imported a watch-only address with the label "cold + wallet" in earlier releases of Bitcoin Core, subsequently importing + the private key would default to resetting the address's label to the + default empty-string label (""). In this release, the previous label + of "cold wallet" will be retained. If you optionally specify any + label besides the default when calling `importprivkey`, the new label + will be applied to the address. + +- See the [Mining](#mining) section for changes to `getblocktemplate`. + +- `getmininginfo` now omits `currentblockweight` and `currentblocktx` + when a block was never assembled via RPC on this node. + +- The `getrawtransaction` RPC & REST endpoints no longer check the + unspent UTXO set for a transaction. The remaining behaviors are as + follows: 1. If a blockhash is provided, check the corresponding block. + 2. If no blockhash is provided, check the mempool. 3. If no blockhash + is provided but txindex is enabled, also check txindex. + +- `unloadwallet` is now synchronous, meaning it will not return until + the wallet is fully unloaded. + +- `importmulti` now supports importing of addresses from descriptors. A + "desc" parameter can be provided instead of the "scriptPubKey" in a + request, as well as an optional range for ranged descriptors to + specify the start and end of the range to import. Descriptors with key + origin information imported through `importmulti` will have their key + origin information stored in the wallet for use with creating PSBTs. + More information about descriptors can be found + [here](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). + +- `listunspent` has been modified so that it also returns + `witnessScript`, the witness script in the case of a P2WSH or + P2SH-P2WSH output. + +- `createwallet` now has an optional `blank` argument that can be used + to create a blank wallet. Blank wallets do not have any keys or HD + seed. They cannot be opened in software older than 0.18. Once a blank + wallet has a HD seed set (by using `sethdseed`) or private keys, + scripts, addresses, and other watch only things have been imported, + the wallet is no longer blank and can be opened in 0.17.x. Encrypting + a blank wallet will also set a HD seed for it. + +Deprecated or removed RPCs +-------------------------- + +- `signrawtransaction` is removed after being deprecated and hidden + behind a special configuration option in version 0.17.0. + +- The 'account' API is removed after being deprecated in v0.17. The + 'label' API was introduced in v0.17 as a replacement for accounts. + See the [release notes from + v0.17](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.17.0.md#label-and-account-apis-for-wallet) + for a full description of the changes from the 'account' API to the + 'label' API. + +- `addwitnessaddress` is removed after being deprecated in version + 0.16.0. + +- `generate` is deprecated and will be fully removed in a subsequent + major version. This RPC is only used for testing, but its + implementation reached across multiple subsystems (wallet and mining), + so it is being deprecated to simplify the wallet-node interface. + Projects that are using `generate` for testing purposes should + transition to using the `generatetoaddress` RPC, which does not + require or use the wallet component. Calling `generatetoaddress` with + an address returned by the `getnewaddress` RPC gives the same + functionality as the old `generate` RPC. To continue using `generate` + in this version, restart bitcoind with the `-deprecatedrpc=generate` + configuration option. + +- Be reminded that parts of the `validateaddress` command have been + deprecated and moved to `getaddressinfo`. The following deprecated + fields have moved to `getaddressinfo`: `ismine`, `iswatchonly`, + `script`, `hex`, `pubkeys`, `sigsrequired`, `pubkey`, `embedded`, + `iscompressed`, `label`, `timestamp`, `hdkeypath`, `hdmasterkeyid`. + +- The `addresses` field has been removed from the `validateaddress` + and `getaddressinfo` RPC methods. This field was confusing since + it referred to public keys using their P2PKH address. Clients + should use the `embedded.address` field for P2SH or P2WSH wrapped + addresses, and `pubkeys` for inspecting multisig participants. + +REST changes +------------ + +- A new `/rest/blockhashbyheight/` endpoint is added for fetching the + hash of the block in the current best blockchain based on its height + (how many blocks it is after the Genesis Block). + +Graphical User Interface (GUI) +------------------------------ + +- A new Window menu is added alongside the existing File, Settings, and + Help menus. Several items from the other menus that opened new + windows have been moved to this new Window menu. + +- In the Send tab, the checkbox for "pay only the required fee" has been + removed. Instead, the user can simply decrease the value in the + Custom Feerate field all the way down to the node's configured minimum + relay fee. + +- In the Overview tab, the watch-only balance will be the only balance + shown if the wallet was created using the `createwallet` RPC and the + `disable_private_keys` parameter was set to true. + +- The launch-on-startup option is no longer available on macOS if + compiled with macosx min version greater than 10.11 (use + CXXFLAGS="-mmacosx-version-min=10.11" + CFLAGS="-mmacosx-version-min=10.11" for setting the deployment sdk + version) + +Tools +----- + +- A new `bitcoin-wallet` tool is now distributed alongside Bitcoin + Core's other executables. Without needing to use any RPCs, this tool + can currently create a new wallet file or display some basic + information about an existing wallet, such as whether the wallet is + encrypted, whether it uses an HD seed, how many transactions it + contains, and how many address book entries it has. + +Planned changes +=============== + +This section describes planned changes to Bitcoin Core that may affect +other Bitcoin software and services. + +- Since version 0.16.0, Bitcoin Core’s built-in wallet has defaulted to + generating P2SH-wrapped segwit addresses when users want to receive + payments. These addresses are backwards compatible with all + widely-used software. Starting with Bitcoin Core 0.20 (expected about + a year after 0.18), Bitcoin Core will default to native segwit + addresses (bech32) that provide additional fee savings and other + benefits. Currently, many wallets and services already support sending + to bech32 addresses, and if the Bitcoin Core project sees enough + additional adoption, it will instead default to bech32 receiving + addresses in Bitcoin Core 0.19 (approximately November 2019). + P2SH-wrapped segwit addresses will continue to be provided if the user + requests them in the GUI or by RPC, and anyone who doesn’t want the + update will be able to configure their default address type. + (Similarly, pioneering users who want to change their default now may + set the `addresstype=bech32` configuration option in any Bitcoin Core + release from 0.16.0 up.) + +Deprecated P2P messages +----------------------- + +- BIP 61 reject messages are now deprecated. Reject messages have no use + case on the P2P network and are only logged for debugging by most + network nodes. Furthermore, they increase bandwidth and can be harmful + for privacy and security. It has been possible to disable BIP 61 + messages since v0.17 with the `-enablebip61=0` option. BIP 61 messages + will be disabled by default in a future version, before being removed + entirely. + +Low-level changes +================= + +This section describes RPC changes mainly useful for testing, mostly not +relevant in production. The changes are mentioned for completeness. + +RPC +--- + +- The `submitblock` RPC previously returned the reason a rejected block + was invalid the first time it processed that block, but returned a + generic "duplicate" rejection message on subsequent occasions it + processed the same block. It now always returns the fundamental + reason for rejecting an invalid block and only returns "duplicate" for + valid blocks it has already accepted. + +- A new `submitheader` RPC allows submitting block headers independently + from their block. This is likely only useful for testing. + +- The `signrawtransactionwithkey` and `signrawtransactionwithwallet` + RPCs have been modified so that they also optionally accept a + `witnessScript`, the witness script in the case of a P2WSH or + P2SH-P2WSH output. This is compatible with the change to + `listunspent`. + +- For the `walletprocesspsbt` and `walletcreatefundedpsbt` RPCs, if the + `bip32derivs` parameter is set to true but the key metadata for a + public key has not been updated yet, then that key will have a + derivation path as if it were just an independent key (i.e. no + derivation path and its master fingerprint is itself). + +Configuration +------------- + +- The `-usehd` configuration option was removed in version 0.16. From + that version onwards, all new wallets created are hierarchical + deterministic wallets. This release makes specifying `-usehd` an + invalid configuration option. + +Network +------- + +- This release allows peers that your node automatically disconnected + for misbehavior (e.g. sending invalid data) to reconnect to your node + if you have unused incoming connection slots. If your slots fill up, + a misbehaving node will be disconnected to make room for nodes without + a history of problems (unless the misbehaving node helps your node in + some other way, such as by connecting to a part of the Internet from + which you don't have many other peers). Previously, Bitcoin Core + banned the IP addresses of misbehaving peers for a period of time + (default of 1 day); this was easily circumvented by attackers with + multiple IP addresses. If you manually ban a peer, such as by using + the `setban` RPC, all connections from that peer will still be + rejected. + +Wallet +------- + +- The key metadata will need to be upgraded the first time that the HD + seed is available. For unencrypted wallets this will occur on wallet + loading. For encrypted wallets this will occur the first time the + wallet is unlocked. + +- Newly encrypted wallets will no longer require restarting the + software. Instead such wallets will be completely unloaded and + reloaded to achieve the same effect. + +- A sub-project of Bitcoin Core now provides Hardware Wallet Interaction + (HWI) scripts that allow command-line users to use several popular + hardware key management devices with Bitcoin Core. See their [project + page](https://github.com/bitcoin-core/HWI#readme) for details. + +Security +-------- + +- This release changes the Random Number Generator (RNG) used from + OpenSSL to Bitcoin Core's own implementation, although entropy + gathered by Bitcoin Core is fed out to OpenSSL and then read back in + when the program needs strong randomness. This moves Bitcoin Core a + little closer to no longer needing to depend on OpenSSL, a dependency + that has caused security issues in the past. The new implementation + gathers entropy from multiple sources, including from hardware + supporting the rdseed CPU instruction. + +Changes for particular platforms +-------------------------------- + +- On macOS, Bitcoin Core now opts out of application CPU throttling + ("app nap") during initial blockchain download, when catching up from + over 100 blocks behind the current chain tip, or when reindexing chain + data. This helps prevent these operations from taking an excessively + long time because the operating system is attempting to conserve + power. + +0.18.0 change log +================= + +### Consensus +- #14247 Fix crash bug with duplicate inputs within a transaction (TheBlueMatt) + +### Mining +- #14811 Mining: Enforce that segwit option must be set in GBT (jnewbery) + +### Block and transaction handling +- #13310 Report progress in ReplayBlocks while rolling forward (promag) +- #13783 validation: Pass tx pool reference into CheckSequenceLocks (MarcoFalke) +- #14834 validation: Assert that pindexPrev is non-null when required (kallewoof) +- #14085 index: Fix for indexers skipping genesis block (jimpo) +- #14963 mempool, validation: Explain `cs_main` locking semantics (MarcoFalke) +- #15193 Default `-whitelistforcerelay` to off (sdaftuar) +- #15429 Update `assumevalid`, `minimumchainwork`, and `getchaintxstats` to height 563378 (gmaxwell) +- #15552 Granular invalidateblock and RewindBlockIndex (MarcoFalke) +- #14841 Move CheckBlock() call to critical section (hebasto) + +### P2P protocol and network code +- #14025 Remove dead code for nVersion=10300 (MarcoFalke) +- #12254 BIP 158: Compact Block Filters for Light Clients (jimpo) +- #14073 blockfilter: Avoid out-of-bounds script access (jimpo) +- #14140 Switch nPrevNodeCount to vNodesSize (pstratem) +- #14027 Skip stale tip checking if outbound connections are off or if reindexing (gmaxwell) +- #14532 Never bind `INADDR_ANY` by default, and warn when doing so explicitly (luke-jr) +- #14733 Make peer timeout configurable, speed up very slow test and ensure correct code path tested (zallarak) +- #14336 Implement poll (pstratem) +- #15051 IsReachable is the inverse of IsLimited (DRY). Includes unit tests (mmachicao) +- #15138 Drop IsLimited in favor of IsReachable (Empact) +- #14605 Return of the Banman (dongcarl) +- #14970 Add dnsseed.emzy.de to DNS seeds (Emzy) +- #14929 Allow connections from misbehavior banned peers (gmaxwell) +- #15345 Correct comparison of addr count (dongcarl) +- #15201 Add missing locking annotation for vNodes. vNodes is guarded by cs_vNodes (practicalswift) +- #14626 Select orphan transaction uniformly for eviction (sipa) +- #15486 Ensure tried collisions resolve, and allow feeler connections to existing outbound netgroups (sdaftuar) + +### Wallet +- #13962 Remove unused `dummy_tx` variable from FillPSBT (dongcarl) +- #13967 Don't report `minversion` wallet entry as unknown (instagibbs) +- #13988 Add checks for settxfee reasonableness (ajtowns) +- #12559 Avoid locking `cs_main` in some wallet RPC (promag) +- #13631 Add CMerkleTx::IsImmatureCoinBase method (Empact) +- #14023 Remove accounts RPCs (jnewbery) +- #13825 Kill accounts (jnewbery) +- #10605 Add AssertLockHeld assertions in CWallet::ListCoins (ryanofsky) +- #12490 Remove deprecated wallet rpc features from `bitcoin_server` (jnewbery) +- #14138 Set `encrypted_batch` to nullptr after delete. Avoid double free in the case of NDEBUG (practicalswift) +- #14168 Remove `ENABLE_WALLET` from `libbitcoin_server.a` (jnewbery) +- #12493 Reopen CDBEnv after encryption instead of shutting down (achow101) +- #14282 Remove `-usehd` option (jnewbery) +- #14146 Remove trailing separators from `-walletdir` arg (PierreRochard) +- #14291 Add ListWalletDir utility function (promag) +- #14468 Deprecate `generate` RPC method (jnewbery) +- #11634 Add missing `cs_wallet`/`cs_KeyStore` locks to wallet (practicalswift) +- #14296 Remove `addwitnessaddress` (jnewbery) +- #14451 Add BIP70 deprecation warning and allow building GUI without BIP70 support (jameshilliard) +- #14320 Fix duplicate fileid detection (ken2812221) +- #14561 Remove `fs::relative` call and fix listwalletdir tests (promag) +- #14454 Add SegWit support to importmulti (MeshCollider) +- #14410 rpcwallet: `ischange` field for `getaddressinfo` RPC (mrwhythat) +- #14350 Add WalletLocation class (promag) +- #14689 Require a public key to be retrieved when signing a P2PKH input (achow101) +- #14478 Show error to user when corrupt wallet unlock fails (MeshCollider) +- #14411 Restore ability to list incoming transactions by label (ryanofsky) +- #14552 Detect duplicate wallet by comparing the db filename (ken2812221) +- #14678 Remove redundant KeyOriginInfo access, already done in CreateSig (instagibbs) +- #14477 Add ability to convert solvability info to descriptor (sipa) +- #14380 Fix assert crash when specified change output spend size is unknown (instagibbs) +- #14760 Log env path in `BerkeleyEnvironment::Flush` (promag) +- #14646 Add expansion cache functions to descriptors (unused for now) (sipa) +- #13076 Fix ScanForWalletTransactions to return an enum indicating scan result: `success` / `failure` / `user_abort` (Empact) +- #14821 Replace CAffectedKeysVisitor with descriptor based logic (sipa) +- #14957 Initialize `stop_block` in CWallet::ScanForWalletTransactions (Empact) +- #14565 Overhaul `importmulti` logic (sipa) +- #15039 Avoid leaking nLockTime fingerprint when anti-fee-sniping (MarcoFalke) +- #14268 Introduce SafeDbt to handle Dbt with free or `memory_cleanse` raii-style (Empact) +- #14711 Remove uses of chainActive and mapBlockIndex in wallet code (ryanofsky) +- #15279 Clarify rescanblockchain doc (MarcoFalke) +- #15292 Remove `boost::optional`-related false positive -Wmaybe-uninitialized warnings on GCC compiler (hebasto) +- #13926 [Tools] bitcoin-wallet - a tool for creating and managing wallets offline (jnewbery) +- #11911 Free BerkeleyEnvironment instances when not in use (ryanofsky) +- #15235 Do not import private keys to wallets with private keys disabled (achow101) +- #15263 Descriptor expansions only need pubkey entries for PKH/WPKH (sipa) +- #15322 Add missing `cs_db` lock (promag) +- #15297 Releases dangling files on `BerkeleyEnvironment::Close` (promag) +- #14491 Allow descriptor imports with importmulti (MeshCollider) +- #15365 Add lock annotation for mapAddressBook (MarcoFalke) +- #15226 Allow creating blank (empty) wallets (alternative) (achow101) +- #15390 [wallet-tool] Close bdb when flushing wallet (jnewbery) +- #15334 Log absolute paths for the wallets (hebasto) +- #14978 Factor out PSBT utilities from RPCs for use in GUI code; related refactoring (gwillen) +- #14481 Add P2SH-P2WSH support to listunspent RPC (MeshCollider) +- #14021 Import key origin data through descriptors in importmulti (achow101) +- #14075 Import watch only pubkeys to the keypool if private keys are disabled (achow101) +- #15368 Descriptor checksums (sipa) +- #15433 Use a single wallet batch for `UpgradeKeyMetadata` (jonasschnelli) +- #15408 Remove unused `TransactionError` constants (MarcoFalke) +- #15583 Log and ignore errors in ListWalletDir and IsBerkeleyBtree (promag) +- #14195 Pass privkey export DER compression flag correctly (fingera) +- #15299 Fix assertion in `CKey::SignCompact` (promag) +- #14437 Start to separate wallet from node (ryanofsky) +- #15749 Fix: importmulti only imports origin info for PKH outputs (sipa) + +### RPC and other APIs +- #12842 Prevent concurrent `savemempool` (promag) +- #13987 Report `minfeefilter` value in `getpeerinfo` RPC (ajtowns) +- #13891 Remove getinfo deprecation warning (jnewbery) +- #13399 Add `submitheader` (MarcoFalke) +- #12676 Show `bip125-replaceable` flag, when retrieving mempool entries (dexX7) +- #13723 PSBT key path cleanups (sipa) +- #14008 Preserve a format of RPC command definitions (kostyantyn) +- #9332 Let wallet `importmulti` RPC accept labels for standard scriptPubKeys (ryanofsky) +- #13983 Return more specific reject reason for submitblock (MarcoFalke) +- #13152 Add getnodeaddresses RPC command (chris-belcher) +- #14298 rest: Improve performance for JSON calls (alecalve) +- #14297 Remove warning for removed estimatefee RPC (jnewbery) +- #14373 Consistency fixes for RPC descriptions (ch4ot1c) +- #14150 Add key origin support to descriptors (sipa) +- #14518 Always throw in getblockstats if `-txindex` is required (promag) +- #14060 ZMQ: add options to configure outbound message high water mark, aka SNDHWM (mruddy) +- #13381 Add possibility to preserve labels on importprivkey (marcoagner) +- #14530 Use `RPCHelpMan` to generate RPC doc strings (MarcoFalke) +- #14720 Correctly name RPC arguments (MarcoFalke) +- #14726 Use `RPCHelpMan` for all RPCs (MarcoFalke) +- #14796 Pass argument descriptions to `RPCHelpMan` (MarcoFalke) +- #14670 http: Fix HTTP server shutdown (promag) +- #14885 Assert that named arguments are unique in `RPCHelpMan` (promag) +- #14877 Document default values for optional arguments (MarcoFalke) +- #14875 RPCHelpMan: Support required arguments after optional ones (MarcoFalke) +- #14993 Fix data race (UB) in InterruptRPC() (practicalswift) +- #14653 rpcwallet: Add missing transaction categories to RPC helptexts (andrewtoth) +- #14981 Clarify RPC `getrawtransaction`'s time help text (benthecarman) +- #12151 Remove `cs_main` lock from blockToJSON and blockheaderToJSON (promag) +- #15078 Document `bytessent_per_msg` and `bytesrecv_per_msg` (MarcoFalke) +- #15057 Correct `reconsiderblock `help text, add test (MarcoFalke) +- #12153 Avoid permanent `cs_main` lock in `getblockheader` (promag) +- #14982 Add `getrpcinfo` command (promag) +- #15122 Expand help text for `importmulti` changes (jnewbery) +- #15186 remove duplicate solvable field from `getaddressinfo` (fanquake) +- #15209 zmq: log outbound message high water mark when reusing socket (fanquake) +- #15177 rest: Improve tests and documention of /headers and /block (promag) +- #14353 rest: Add blockhash call, fetch blockhash by height (jonasschnelli) +- #15248 Compile on GCC4.8 (MarcoFalke) +- #14987 RPCHelpMan: Pass through Result and Examples (MarcoFalke) +- #15159 Remove lookup to UTXO set from GetTransaction (amitiuttarwar) +- #15245 remove deprecated mentions of signrawtransaction from fundraw help (instagibbs) +- #14667 Add `deriveaddresses` RPC util method (Sjors) +- #15357 Don't ignore `-maxtxfee` when wallet is disabled (JBaczuk) +- #15337 Fix for segfault if combinepsbt called with empty inputs (benthecarman) +- #14918 RPCHelpMan: Check default values are given at compile-time (MarcoFalke) +- #15383 mining: Omit uninitialized currentblockweight, currentblocktx (MarcoFalke) +- #13932 Additional utility RPCs for PSBT (achow101) +- #15401 Actually throw help when passed invalid number of params (MarcoFalke) +- #15471 rpc/gui: Remove 'Unknown block versions being mined' warning (laanwj) +- #15497 Consistent range arguments in scantxoutset/importmulti/deriveaddresses (sipa) +- #15510 deriveaddresses: add range to CRPCConvertParam (Sjors) +- #15582 Fix overflow bug in analyzepsbt fee: CAmount instead of int (sipa) +- #13424 Consistently validate txid / blockhash length and encoding in rpc calls (Empact) +- #15750 Remove the addresses field from the getaddressinfo return object (jnewbery) + +### GUI +- #13634 Compile `boost::signals2` only once (MarcoFalke) +- #13248 Make proxy icon from statusbar clickable (mess110) +- #12818 TransactionView: highlight replacement tx after fee bump (Sjors) +- #13529 Use new Qt5 connect syntax (promag) +- #14162 Also log and print messages or questions like bitcoind (MarcoFalke) +- #14385 Avoid system harfbuzz and bz2 (theuni) +- #14450 Fix QCompleter popup regression (hebasto) +- #14177 Set C locale for amountWidget (hebasto) +- #14374 Add `Blocksdir` to Debug window (hebasto) +- #14554 Remove unused `adjustedTime` parameter (hebasto) +- #14228 Enable system tray icon by default if available (hebasto) +- #14608 Remove the "Pay only required fee…" checkbox (hebasto) +- #14521 qt, docs: Fix `bitcoin-qt -version` output formatting (hebasto) +- #13966 When private key is disabled, only show watch-only balance (ken2812221) +- #14828 Remove hidden columns in coin control dialog (promag) +- #14783 Fix `boost::signals2::no_slots_error` in early calls to InitWarning (promag) +- #14854 Cleanup SplashScreen class (hebasto) +- #14801 Use window() instead of obsolete topLevelWidget() (hebasto) +- #14573 Add Window menu (promag) +- #14979 Restore < Qt5.6 compatibility for addAction (jonasschnelli) +- #14975 Refactoring with QString::toNSString() (hebasto) +- #15000 Fix broken notificator on GNOME (hebasto) +- #14375 Correct misleading "overridden options" label (hebasto) +- #15007 Notificator class refactoring (hebasto) +- #14784 Use `WalletModel*` instead of the wallet name as map key (promag) +- #11625 Add BitcoinApplication & RPCConsole tests (ryanofsky) +- #14517 Fix start with the `-min` option (hebasto) +- #13216 implements concept for different disk sizes on intro (marcoagner) +- #15114 Replace remaining 0 with nullptr (Empact) +- #14594 Fix minimized window bug on Linux (hebasto) +- #14556 Fix confirmed transaction labeled "open" (#13299) (hebasto) +- #15149 Show current wallet name in window title (promag) +- #15136 "Peers" tab overhaul (hebasto) +- #14250 Remove redundant stopThread() and stopExecutor() signals (hebasto) +- #15040 Add workaround for QProgressDialog bug on macOS (hebasto) +- #15101 Add WalletController (promag) +- #15178 Improve "help-console" message (hebasto) +- #15210 Fix window title update (promag) +- #15167 Fix wallet selector size adjustment (hebasto) +- #15208 Remove macOS launch-at-startup when compiled with > macOS 10.11, fix memory mismanagement (jonasschnelli) +- #15163 Correct units for "-dbcache" and "-prune" (hebasto) +- #15225 Change the receive button to respond to keypool state changing (achow101) +- #15280 Fix shutdown order (promag) +- #15203 Fix issue #9683 "gui, wallet: random abort (segmentation fault) (dooglus) +- #15091 Fix model overlay header sync (jonasschnelli) +- #15153 Add Open Wallet menu (promag) +- #15183 Fix `m_assumed_blockchain_size` variable value (marcoagner) +- #15063 If BIP70 is disabled, attempt to fall back to BIP21 parsing (luke-jr) +- #15195 Add Close Wallet action (promag) +- #15462 Fix async open wallet call order (promag) +- #15801 Bugfix: GUI: Options: Initialise prune setting range before loading current value, and remove upper bound limit (luke-jr) + +### Build system +- #13955 gitian: Bump descriptors for (0.)18 (fanquake) +- #13899 Enable -Wredundant-decls where available. Remove redundant redeclarations (practicalswift) +- #13665 Add RISC-V support to gitian (ken2812221) +- #14062 Generate MSVC project files via python script (ken2812221) +- #14037 Add README.md to linux release tarballs (hebasto) +- #14183 Remove unused Qt 4 dependencies (ken2812221) +- #14127 Avoid getifaddrs when unavailable (greenaddress) +- #14184 Scripts and tools: increased timeout downloading (cisba) +- #14204 Move `interfaces/*` to `libbitcoin_server` (laanwj) +- #14208 Actually remove `ENABLE_WALLET` (jnewbery) +- #14212 Remove libssl from LDADD unless GUI (MarcoFalke) +- #13578 Upgrade zeromq to 4.2.5 and avoid deprecated zeromq API functions (mruddy) +- #14281 lcov: filter /usr/lib/ from coverage reports (MarcoFalke) +- #14325 gitian: Use versioned unsigned tarballs instead of generically named ones (achow101) +- #14253 During 'make clean', remove some files that are currently missed (murrayn) +- #14455 Unbreak `make clean` (jamesob) +- #14495 Warn (don't fail!) on spelling errors (practicalswift) +- #14496 Pin to specific versions of Python packages we install from PyPI in Travis (practicalswift) +- #14568 Fix Qt link order for Windows build (ken2812221) +- #14252 Run functional tests and benchmarks under the undefined behaviour sanitizer (UBSan) (practicalswift) +- #14612 Include full version number in released file names (achow101) +- #14840 Remove duplicate libconsensus linking in test make (AmirAbrams) +- #14564 Adjust configure so that only BIP70 is disabled when protobuf is missing instead of the GUI (jameshilliard) +- #14883 Add `--retry 5` to curl opts in `install_db4.sh` (qubenix) +- #14701 Add `CLIENT_VERSION_BUILD` to CFBundleGetInfoString (fanquake) +- #14849 Qt 5.9.7 (fanquake) +- #15020 Add names to Travis jobs (gkrizek) +- #15047 Allow to configure --with-sanitizers=fuzzer (MarcoFalke) +- #15154 Configure: bitcoin-tx doesn't need libevent, so don't pull it in (luke-jr) +- #15175 Drop macports support (Empact) +- #15308 Restore compatibility with older boost (Empact) +- #15407 msvc: Fix silent merge conflict between #13926 and #14372 part II (ken2812221) +- #15388 Makefile.am: add rule for src/bitcoin-wallet (Sjors) +- #15393 Bump minimum Qt version to 5.5.1 (Sjors) +- #15285 Prefer Python 3.4 even if newer versions are present on the system (Sjors) +- #15398 msvc: Add rapidcheck property tests (ken2812221) +- #15431 msvc: scripted-diff: Remove NDEBUG pre-define in project file (ken2812221) +- #15549 gitian: Improve error handling (laanwj) +- #15548 use full version string in setup.exe (MarcoFalke) +- #11526 Visual Studio build configuration for Bitcoin Core (sipsorcery) +- #15110 build\_msvc: Fix the build problem in `libbitcoin_server` (Mr-Leshiy) +- #14372 msvc: build secp256k1 and leveldb locally (ken2812221) +- #15325 msvc: Fix silent merge conflict between #13926 and #14372 (ken2812221) +- #15391 Add compile time verification of assumptions we're currently making implicitly/tacitly (practicalswift) +- #15503 msvc: Use a single file to specify the include path (ken2812221) +- #13765 contrib: Add gitian build support for github pull request (ken2812221) +- #15809 gitignore: plist and dat (jamesob) + +### Tests and QA +- #15405 appveyor: Clean cache when build configuration changes (Sjors) +- #13953 Fix deprecation in bitcoin-util-test.py (isghe) +- #13963 Replace usage of tostring() with tobytes() (dongcarl) +- #13964 ci: Add appveyor ci (ken2812221) +- #13997 appveyor: fetch the latest port data (ken2812221) +- #13707 Add usage note to check-rpc-mappings.py (masonicboom) +- #14036 travis: Run unit tests --with-sanitizers=undefined (MarcoFalke) +- #13861 Add testing of `value_ret` for SelectCoinsBnB (Empact) +- #13863 travis: Move script sections to files in `.travis/` subject to shellcheck (scravy) +- #14081 travis: Fix missing differentiation between unit and functional tests (scravy) +- #14042 travis: Add cxxflags=-wno-psabi at arm job (ken2812221) +- #14051 Make `combine_logs.py` handle multi-line logs (jnewbery) +- #14093 Fix accidental trunction from int to bool (practicalswift) +- #14108 Add missing locking annotations and locks (`g_cs_orphans`) (practicalswift) +- #14088 Don't assert(…) with side effects (practicalswift) +- #14086 appveyor: Use clcache to speed up build (ken2812221) +- #13954 Warn (don't fail!) on spelling errors. Fix typos reported by codespell (practicalswift) +- #12775 Integration of property based testing into Bitcoin Core (Christewart) +- #14119 Read reject reasons from debug log, not P2P messages (MarcoFalke) +- #14189 Fix silent merge conflict in `wallet_importmulti` (MarcoFalke) +- #13419 Speed up `knapsack_solver_test` by not recreating wallet 100 times (lucash-dev) +- #14199 Remove redundant BIP174 test from `rpc_psbt.json` (araspitzu) +- #14179 Fixups to "Run all tests even if wallet is not compiled" (MarcoFalke) +- #14225 Reorder tests and move most of extended tests up to normal tests (ken2812221) +- #14236 `generate` --> `generatetoaddress` change to allow tests run without wallet (sanket1729) +- #14287 Use MakeUnique to construct objects owned by `unique_ptrs` (practicalswift) +- #14007 Run functional test on Windows and enable it on Appveyor (ken2812221) +- #14275 Write the notification message to different files to avoid race condition in `feature_notifications.py` (ken2812221) +- #14306 appveyor: Move AppVeyor YAML to dot-file-style YAML (MitchellCash) +- #14305 Enforce critical class instance attributes in functional tests, fix segwit test specificity (JustinTArthur) +- #12246 Bugfix: Only run bitcoin-tx tests when bitcoin-tx is enabled (luke-jr) +- #14316 Exclude all tests with difference parameters in `--exclude` list (ken2812221) +- #14381 Add missing call to `skip_if_no_cli()` (practicalswift) +- #14389 travis: Set codespell version to avoid breakage (MarcoFalke) +- #14398 Don't access out of bounds array index: array[sizeof(array)] (Empact) +- #14419 Remove `rpc_zmq.py` (jnewbery) +- #14241 appveyor: Script improvement (ken2812221) +- #14413 Allow closed RPC handler in `assert_start_raises_init_error` (ken2812221) +- #14324 Run more tests with wallet disabled (MarcoFalke) +- #13649 Allow arguments to be forwarded to flake8 in lint-python.sh (jamesob) +- #14465 Stop node before removing the notification file (ken2812221) +- #14460 Improve 'CAmount' tests (hebasto) +- #14456 forward timeouts properly in `send_blocks_and_test` (jamesob) +- #14527 Revert "Make qt wallet test compatible with qt4" (MarcoFalke) +- #14504 Show the progress of functional tests (isghe) +- #14559 appveyor: Enable multiwallet tests (ken2812221) +- #13515 travis: Enable qt for all jobs (ken2812221) +- #14571 Test that nodes respond to `getdata` with `notfound` (MarcoFalke) +- #14569 Print dots by default in functional tests (ken2812221) +- #14631 Move deterministic address import to `setup_nodes` (jnewbery) +- #14630 test: Remove travis specific code (MarcoFalke) +- #14528 travis: Compile once on xenial (MarcoFalke) +- #14092 Dry run `bench_bitcoin` as part `make check` to allow for quick identification of assertion/sanitizer failures in benchmarking code (practicalswift) +- #14664 `example_test.py`: fixup coinbase height argument, derive number clearly (instagibbs) +- #14522 Add invalid P2P message tests (jamesob) +- #14619 Fix value display name in `test_runner` help text (merland) +- #14672 Send fewer spam messages in `p2p_invalid_messages` (jamesob) +- #14673 travis: Fail the ubsan travis build in case of newly introduced ubsan errors (practicalswift) +- #14665 appveyor: Script improvement part II (ken2812221) +- #14365 Add Python dead code linter (vulture) to Travis (practicalswift) +- #14693 `test_node`: `get_mem_rss` fixups (MarcoFalke) +- #14714 util.h: explicitly include required QString header (1Il1) +- #14705 travis: Avoid timeout on verify-commits check (MarcoFalke) +- #14770 travis: Do not specify sudo in `.travis` (scravy) +- #14719 Check specific reject reasons in `feature_block` (MarcoFalke) +- #14771 Add `BOOST_REQUIRE` to getters returning optional (MarcoFalke) +- #14777 Add regtest for JSON-RPC batch calls (domob1812) +- #14764 travis: Run thread sanitizer on unit tests (MarcoFalke) +- #14400 Add Benchmark to test input de-duplication worst case (JeremyRubin) +- #14812 Fix `p2p_invalid_messages` on macOS (jamesob) +- #14813 Add `wallet_encryption` error tests (MarcoFalke) +- #14820 Fix `descriptor_tests` not checking ToString output of public descriptors (ryanofsky) +- #14794 Add AddressSanitizer (ASan) Travis build (practicalswift) +- #14819 Bugfix: `test/functional/mempool_accept`: Ensure oversize transaction is actually oversize (luke-jr) +- #14822 bench: Destroy wallet txs instead of leaking their memory (MarcoFalke) +- #14683 Better `combine_logs.py` behavior (jamesob) +- #14231 travis: Save cache even when build or test fail (ken2812221) +- #14816 Add CScriptNum decode python implementation in functional suite (instagibbs) +- #14861 Modify `rpc_bind` to conform to #14532 behaviour (dongcarl) +- #14864 Run scripted-diff in subshell (dongcarl) +- #14795 Allow `test_runner` command line to receive parameters for each test (marcoagner) +- #14788 Possible fix the permission error when the tests open the cookie file (ken2812221) +- #14857 `wallet_keypool_topup.py`: Test for all keypool address types (instagibbs) +- #14886 Refactor importmulti tests (jnewbery) +- #14908 Removed implicit CTransaction constructor calls from tests and benchmarks (lucash-dev) +- #14903 Handle ImportError explicitly, improve comparisons against None (daniel-s-ingram) +- #14884 travis: Enforce python 3.4 support through linter (Sjors) +- #14940 Add test for truncated pushdata script (MarcoFalke) +- #14926 consensus: Check that final transactions are valid (MarcoFalke) +- #14937 travis: Fix travis would always be green even if it fail (ken2812221) +- #14953 Make `g_insecure_rand_ctx` `thread_local` (MarcoFalke) +- #14931 mempool: Verify prioritization is dumped correctly (MarcoFalke) +- #14935 Test for expected return values when calling functions returning a success code (practicalswift) +- #14969 Fix `cuckoocache_tests` TSAN failure introduced in 14935 (practicalswift) +- #14964 Fix race in `mempool_accept` (MarcoFalke) +- #14829 travis: Enable functional tests in the threadsanitizer (tsan) build job (practicalswift) +- #14985 Remove `thread_local` from `test_bitcoin` (MarcoFalke) +- #15005 Bump timeout to run tests in travis thread sanitizer (MarcoFalke) +- #15013 Avoid race in `p2p_timeouts` (MarcoFalke) +- #14960 lint/format-strings: Correctly exclude escaped percent symbols (luke-jr) +- #14930 pruning: Check that verifychain can be called when pruned (MarcoFalke) +- #15022 Upgrade Travis OS to Xenial (gkrizek) +- #14738 Fix running `wallet_listtransactions.py` individually through `test_runner.py` (kristapsk) +- #15026 Rename `rpc_timewait` to `rpc_timeout` (MarcoFalke) +- #15069 Fix `rpc_net.py` `pong` race condition (Empact) +- #14790 Allow running `rpc_bind.py` --nonloopback test without IPv6 (kristapsk) +- #14457 add invalid tx templates for use in functional tests (jamesob) +- #14855 Correct ineffectual WithOrVersion from `transactions_tests` (Empact) +- #15099 Use `std::vector` API for construction of test data (domob1812) +- #15102 Run `invalid_txs.InputMissing` test in `feature_block` (MarcoFalke) +- #15059 Add basic test for BIP34 (MarcoFalke) +- #15108 Tidy up `wallet_importmulti.py` (amitiuttarwar) +- #15164 Ignore shellcheck warning SC2236 (promag) +- #15170 refactor/lint: Add ignored shellcheck suggestions to an array (koalaman) +- #14958 Remove race between connecting and shutdown on separate connections (promag) +- #15166 Pin shellcheck version (practicalswift) +- #15196 Update all `subprocess.check_output` functions to be Python 3.4 compatible (gkrizek) +- #15043 Build fuzz targets into seperate executables (MarcoFalke) +- #15276 travis: Compile once on trusty (MarcoFalke) +- #15246 Add tests for invalid message headers (MarcoFalke) +- #15301 When testing with --usecli, unify RPC arg to cli arg conversion and handle dicts and lists (achow101) +- #15247 Use wallet to retrieve raw transactions (MarcoFalke) +- #15303 travis: Remove unused `functional_tests_config` (MarcoFalke) +- #15330 Fix race in `p2p_invalid_messages` (MarcoFalke) +- #15324 Make bloom tests deterministic (MarcoFalke) +- #15328 travis: Revert "run extended tests once daily" (MarcoFalke) +- #15327 Make test `updatecoins_simulation_test` deterministic (practicalswift) +- #14519 add utility to easily profile node performance with perf (jamesob) +- #15349 travis: Only exit early if compilation took longer than 30 min (MarcoFalke) +- #15350 Drop RPC connection if --usecli (promag) +- #15370 test: Remove unused --force option (MarcoFalke) +- #14543 minor `p2p_sendheaders` fix of height in coinbase (instagibbs) +- #13787 Test for Windows encoding issue (ken2812221) +- #15378 Added missing tests for RPC wallet errors (benthecarman) +- #15238 remove some magic mining constants in functional tests (instagibbs) +- #15411 travis: Combine --disable-bip70 into existing job (MarcoFalke) +- #15295 fuzz: Add `test/fuzz/test_runner.py` and run it in travis (MarcoFalke) +- #15413 Add missing `cs_main` locks required when accessing pcoinsdbview, pcoinsTip or pblocktree (practicalswift) +- #15399 fuzz: Script validation flags (MarcoFalke) +- #15410 txindex: interrupt threadGroup before calling destructor (MarcoFalke) +- #15397 Remove manual byte editing in `wallet_tx_clone` func test (instagibbs) +- #15415 functional: allow custom cwd, use tmpdir as default (Sjors) +- #15404 Remove `-txindex` to start nodes (amitiuttarwar) +- #15439 remove `byte.hex()` to keep compatibility (AkioNak) +- #15419 Always refresh cache to be out of ibd (MarcoFalke) +- #15507 Bump timeout on tests that timeout on windows (MarcoFalke) +- #15506 appveyor: fix cache issue and reduce dependencies build time (ken2812221) +- #15485 add `rpc_misc.py`, mv test getmemoryinfo, add test mallocinfo (adamjonas) +- #15321 Add `cs_main` lock annotations for mapBlockIndex (MarcoFalke) +- #14128 lint: Make sure we read the command line inputs using UTF-8 decoding in python (ken2812221) +- #14115 lint: Make all linters work under the default macos dev environment (build-osx.md) (practicalswift) +- #15219 lint: Enable python linters via an array (Empact) + +### Platform support +- #13866 utils: Use `_wfopen` and `_wfreopen` on windows (ken2812221) +- #13886 utils: Run commands using UTF-8 string on windows (ken2812221) +- #14192 utils: Convert `fs::filesystem_error` messages from local multibyte to UTF-8 on windows (ken2812221) +- #13877 utils: Make fs::path::string() always return UTF-8 string on windows (ken2812221) +- #13883 utils: Convert windows args to UTF-8 string (ken2812221) +- #13878 utils: Add fstream wrapper to allow to pass unicode filename on windows (ken2812221) +- #14426 utils: Fix broken windows filelock (ken2812221) +- #14686 Fix windows build error if `--disable-bip70` (ken2812221) +- #14922 windows: Set `_WIN32_WINNT` to 0x0601 (Windows 7) (ken2812221) +- #13888 Call unicode API on Windows (ken2812221) +- #15468 Use `fsbridge::ifstream` to fix Windows path issue (ken2812221) +- #13734 Drop `boost::scoped_array` and use `wchar_t` API explicitly on Windows (ken2812221) +- #13884 Enable bdb unicode support for Windows (ken2812221) + +### Miscellaneous +- #13935 contrib: Adjust output to current test format (AkioNak) +- #14097 validation: Log FormatStateMessage on ConnectBlock error in ConnectTip (MarcoFalke) +- #13724 contrib: Support ARM and RISC-V symbol check (ken2812221) +- #13159 Don't close old debug log file handle prematurely when trying to re-open (on SIGHUP) (practicalswift) +- #14186 bitcoin-cli: don't translate command line options (HashUnlimited) +- #14057 logging: Only log `using config file path_to_bitcoin.conf` message on startup if conf file exists (leishman) +- #14164 Update univalue subtree (MarcoFalke) +- #14272 init: Remove deprecated args from hidden args (MarcoFalke) +- #14494 Error if # is used in rpcpassword in conf (MeshCollider) +- #14742 Properly generate salt in rpcauth.py (dongcarl) +- #14708 Warn unrecognised sections in the config file (AkioNak) +- #14756 Improve rpcauth.py by using argparse and getpass modules (promag) +- #14785 scripts: Fix detection of copyright holders (cornelius) +- #14831 scripts: Use `#!/usr/bin/env bash` instead of `#!/bin/bash` (vim88) +- #14869 Scripts: Add trusted key for samuel dobson (laanwj) +- #14809 Tools: improve verify-commits.py script (jlopp) +- #14624 Some simple improvements to the RNG code (sipa) +- #14947 scripts: Remove python 2 import workarounds (practicalswift) +- #15087 Error if rpcpassword contains hash in conf sections (MeshCollider) +- #14433 Add checksum in gitian build scripts for ossl (TheCharlatan) +- #15165 contrib: Allow use of github api authentication in github-merge (laanwj) +- #14409 utils and libraries: Make 'blocksdir' always net specific (hebasto) +- #14839 threads: Fix unitialized members in `sched_param` (fanquake) +- #14955 Switch all RNG code to the built-in PRNG (sipa) +- #15258 Scripts and tools: Fix `devtools/copyright_header.py` to always honor exclusions (Empact) +- #12255 Update bitcoin.service to conform to init.md (dongcarl) +- #15266 memory: Construct globals on first use (MarcoFalke) +- #15347 Fix build after pr 15266 merged (hebasto) +- #15351 Update linearize-hashes.py (OverlordQ) +- #15358 util: Add setuphelpoptions() (MarcoFalke) +- #15216 Scripts and tools: Replace script name with a special parameter (hebasto) +- #15250 Use RdSeed when available, and reduce RdRand load (sipa) +- #15278 Improve PID file error handling (hebasto) +- #15270 Pull leveldb subtree (MarcoFalke) +- #15456 Enable PID file creation on WIN (riordant) +- #12783 macOS: disable AppNap during sync (krab) +- #13910 Log progress while verifying blocks at level 4 (domob1812) +- #15124 Fail AppInitMain if either disk space check fails (Empact) +- #15117 Fix invalid memory write in case of failing mmap(…) in PosixLockedPageAllocator::AllocateLocked (practicalswift) +- #14357 streams: Fix broken `streams_vector_reader` test. Remove unused `seek(size_t)` +- #11640 Make `LOCK`, `LOCK2`, `TRY_LOCK` work with CWaitableCriticalSection (ryanofsky) +- #14074 Use `std::unordered_set` instead of `set` in blockfilter interface (jimpo) +- #15275 Add gitian PGP key for hebasto (hebasto) + +### Documentation +- #14120 Notes about control port and read access to cookie (JBaczuk) +- #14135 correct GetDifficulty doc after #13288 (fanquake) +- #14013 Add new regtest ports in man following #10825 ports reattributions (ariard) +- #14149 Remove misleading checkpoints comment in CMainParams (MarcoFalke) +- #14153 Add disable-wallet section to OSX build instructions, update line in Unix instructions (bitstein) +- #13662 Explain when reindex-chainstate can be used instead of reindex (Sjors) +- #14207 `-help-debug` implies `-help` (laanwj) +- #14213 Fix reference to lint-locale-dependence.sh (hebasto) +- #14206 Document `-checklevel` levels (laanwj) +- #14217 Add GitHub PR template (MarcoFalke) +- #14331 doxygen: Fix member comments (MarcoFalke) +- #14264 Split depends installation instructions per arch (MarcoFalke) +- #14393 Add missing apt-get install (poiuty) +- #14428 Fix macOS files description in qt/README.md (hebasto) +- #14390 release process: RPC documentation (karel-3d) +- #14472 getblocktemplate: use SegWit in example (Sjors) +- #14497 Add doc/bitcoin-conf.md (hebasto) +- #14526 Document lint tests (fanquake) +- #14511 Remove explicit storage requirement from README.md (merland) +- #14600 Clarify commit message guidelines (merland) +- #14617 FreeBSD: Document Python 3 requirement for 'gmake check' (murrayn) +- #14592 Add external interface consistency guarantees (MarcoFalke) +- #14625 Make clear function argument case in dev notes (dongcarl) +- #14515 Update OpenBSD build guide for 6.4 (fanquake) +- #14436 Add comment explaining recentRejects-DoS behavior (jamesob) +- #14684 conf: Remove deprecated options from docs, Other cleanup (MarcoFalke) +- #14731 Improve scripted-diff developer docs (dongcarl) +- #14778 A few minor formatting fixes and clarifications to descriptors.md (jnewbery) +- #14448 Clarify rpcwallet flag url change (JBaczuk) +- #14808 Clarify RPC rawtransaction documentation (jlopp) +- #14804 Less confusing documentation for `torpassword` (fanquake) +- #14848 Fix broken Gmane URL in security-check.py (cyounkins-bot) +- #14882 developer-notes.md: Point out that UniValue deviates from upstream (Sjors) +- #14909 Update minimum required Qt (fanquake) +- #14914 Add nice table to files.md (emilengler) +- #14741 Indicate `-rpcauth` option password hashing alg (dongcarl) +- #14950 Add NSIS setup/install steps to windows docs (fanquake) +- #13930 Better explain GetAncestor check for `m_failed_blocks` in AcceptBlockHeader (Sjors) +- #14973 Improve Windows native build instructions (murrayn) +- #15073 Botbot.me (IRC logs) not available anymore (anduck) +- #15038 Get more info about GUI-related issue on Linux (hebasto) +- #14832 Add more Doxygen information to Developer Notes (ch4ot1c) +- #15128 Fix download link in doc/README.md (merland) +- #15127 Clarifying testing instructions (benthecarman) +- #15132 Add FreeBSD build notes link to doc/README.md (fanquake) +- #15173 Explain what .python-version does (Sjors) +- #15223 Add information about security to the JSON-RPC doc (harding) +- #15249 Update python docs to reflect that wildcard imports are disallowed (Empact) +- #15176 Get rid of badly named `doc/README_osx.md` (merland) +- #15272 Correct logging return type and RPC example (fanquake) +- #15244 Gdb attaching to process during tests has non-sudo solution (instagibbs) +- #15332 Small updates to `getrawtransaction` description (amitiuttarwar) +- #15354 Add missing `bitcoin-wallet` tool manpages (MarcoFalke) +- #15343 netaddress: Make IPv4 loopback comment more descriptive (dongcarl) +- #15353 Minor textual improvements in `translation_strings_policy.md` (merland) +- #15426 importmulti: add missing description of keypool option (harding) +- #15425 Add missing newline to listunspent help for witnessScript (harding) +- #15348 Add separate productivity notes document (dongcarl) +- #15416 Update FreeBSD build guide for 12.0 (fanquake) +- #15222 Add info about factors that affect dependency list (merland) +- #13676 Explain that mempool memory is added to `-dbcache` (Sjors) +- #15273 Slight tweak to the verify-commits script directions (droark) +- #15477 Remove misleading hint in getrawtransaction (MarcoFalke) +- #15489 Update release process for snap package (MarcoFalke) +- #15524 doc: Remove berkeleydb PPA from linux build instructions (MarcoFalke) +- #15559 Correct `analyzepsbt` rpc doc (fanquake) +- #15194 Add comment describing `fDisconnect` behavior (dongcarl) +- #15754 getrpcinfo docs (benthecarman) +- #15763 Update bips.md for 0.18.0 (sipa) +- #15757 List new RPCs in psbt.md and descriptors.md (sipa) +- #15765 correct bitcoinconsensus_version in shared-libraries.md (fanquake) +- #15792 describe onlynet option in doc/tor.md (jonatack) +- #15802 mention creating application support bitcoin folder on OSX (JimmyMow) +- #15799 Clarify RPC versioning (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 1Il1 +- 251 +- Aaron Clauson +- Adam Jonas +- Akio Nakamura +- Alexander Leishman +- Alexey Ivanov +- Alexey Poghilenkov +- Amir Abrams +- Amiti Uttarwar +- Andrew Chow +- andrewtoth +- Anthony Towns +- Antoine Le Calvez +- Antoine Riard +- Antti Majakivi +- araspitzu +- Arvid Norberg +- Ben Carman +- Ben Woosley +- benthecarman +- bitcoinhodler +- Carl Dong +- Chakib Benziane +- Chris Moore +- Chris Stewart +- chris-belcher +- Chun Kuan Lee +- Cornelius Schumacher +- Cory Fields +- Craig Younkins +- Cristian Mircea Messel +- Damian Mee +- Daniel Ingram +- Daniel Kraft +- David A. Harding +- DesWurstes +- dexX7 +- Dimitri Deijs +- Dimitris Apostolou +- Douglas Roark +- DrahtBot +- Emanuele Cisbani +- Emil Engler +- Eric Scrivner +- fridokus +- Gal Buki +- Gleb Naumenko +- Glenn Willen +- Graham Krizek +- Gregory Maxwell +- Gregory Sanders +- gustavonalle +- Harry Moreno +- Hennadii Stepanov +- Isidoro Ghezzi +- Jack Mallers +- James Hilliard +- James O'Beirne +- Jameson Lopp +- Jeremy Rubin +- Jesse Cohen +- Jim Posen +- John Newbery +- Jon Layton +- Jonas Schnelli +- João Barbosa +- Jordan Baczuk +- Jorge Timón +- Julian Fleischer +- Justin Turner Arthur +- Karel BÃlek +- Karl-Johan Alm +- Kaz Wesley +- ken2812221 +- Kostiantyn Stepaniuk +- Kristaps Kaupe +- Lawrence Nahum +- Lenny Maiorani +- liuyujun +- lucash-dev +- luciana +- Luke Dashjr +- marcaiaf +- marcoagner +- MarcoFalke +- Martin Erlandsson +- Marty Jones +- Mason Simon +- Michael Ford +- Michael Goldstein +- Michael Polzer +- Mitchell Cash +- mruddy +- Murray Nesbitt +- OverlordQ +- Patrick Strateman +- Pierre Rochard +- Pieter Wuille +- poiuty +- practicalswift +- priscoan +- qubenix +- riordant +- Russell Yanofsky +- Samuel Dobson +- sanket1729 +- Sjors Provoost +- Stephan Oeste +- Steven Roose +- Suhas Daftuar +- TheCharlatan +- Tim Ruffing +- Vidar Holen +- vim88 +- Walter +- whythat +- Wladimir J. van der Laan +- Zain Iqbal Allarakhia + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-notes/release-notes-pr12255.md b/doc/release-notes/release-notes-pr12255.md deleted file mode 100644 index 5ac8b44283..0000000000 --- a/doc/release-notes/release-notes-pr12255.md +++ /dev/null @@ -1,17 +0,0 @@ -systemd init file -========= - -The systemd init file (`contrib/init/bitcoind.service`) has been changed to use -`/var/lib/bitcoind` as the data directory instead of `~bitcoin/.bitcoin`. This -change makes Bitcoin Core more consistent with other services, and makes the -systemd init config more consistent with existing Upstart and OpenRC configs. - -The configuration, PID, and data directories are now completely managed by -systemd, which will take care of their creation, permissions, etc. See -[`systemd.exec (5)`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=) -for more details. - -When using the provided init files under `contrib/init`, overriding the -`datadir` option in `/etc/bitcoin/bitcoin.conf` will have no effect. This is -because the command line arguments specified in the init files take precedence -over the options specified in `/etc/bitcoin/bitcoin.conf`. diff --git a/doc/release-process.md b/doc/release-process.md index eb1f5ad222..7522310ce2 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -4,15 +4,14 @@ Release Process Before every release candidate: * Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). - * Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). -* Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`) +* Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). Before every minor and major release: * Update [bips.md](bips.md) to account for changes since the last release. -* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true`) (don't forget to set `CLIENT_VERSION_RC` to `0`) -* Write release notes (see below) +* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). +* Write release notes (see below). * Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. * Update `src/chainparams.cpp` defaultAssumeValid with information from the getblockhash rpc. - The selected value must not be orphaned so it may be useful to set the value two blocks back from the tip. @@ -26,7 +25,13 @@ Before every major release: * Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead. * Update `src/chainparams.cpp` chainTxData with statistics about the transaction count and rate. Use the output of the RPC `getchaintxstats`, see [this pull request](https://github.com/bitcoin/bitcoin/pull/12270) for an example. Reviewers can verify the results by running `getchaintxstats <window_block_count> <window_last_block_hash>` with the `window_block_count` and `window_last_block_hash` from your output. -* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release +* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release. +* In `configure.ac` and `build_msvc/bitcoin_config.h` on _the master branch_: + - update `CLIENT_VERSION_MINOR` version +* In `configure.ac` and `build_msvc/bitcoin_config.h` on _a new release branch_ (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd972fca3dd4a33cfff90bf901b71a687e7)): + - update `CLIENT_VERSION_MINOR` version + - set `CLIENT_VERSION_REVISION` to `0` + - set `CLIENT_VERSION_IS_RELEASE` to `true` ### First time / New builders diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md index dc363582cc..e960863a80 100644 --- a/doc/shared-libraries.md +++ b/doc/shared-libraries.md @@ -7,11 +7,11 @@ The purpose of this library is to make the verification functionality that is cr ### API -The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. +The interface is defined in the C header `bitcoinconsensus.h` located in `src/script/bitcoinconsensus.h`. #### Version -`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently at an experimental `0`)*. +`bitcoinconsensus_version` returns an `unsigned int` with the API version *(currently `1`)*. #### Script Validation diff --git a/doc/tor.md b/doc/tor.md index c46b7e9f60..cfb7f16666 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -16,7 +16,7 @@ outgoing connections, but more is possible. -onion=ip:port Set the proxy server to use for Tor hidden services. You do not need to set this if it's the same as -proxy. You can use -noonion - to explicitly disable access to hidden service. + to explicitly disable access to hidden services. -listen When using -proxy, listening is disabled by default. If you want to run a hidden service (see next section), you'll need to enable @@ -27,6 +27,11 @@ outgoing connections, but more is possible. -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with other P2P nodes. + -onlynet=onion Make outgoing connections only to .onion addresses. Incoming + connections are not affected by this option. This option can be + specified multiple times to allow multiple network types, e.g. + ipv4, ipv6, or onion. + In a typical situation, this suffices to run behind a Tor proxy: ./bitcoind -proxy=127.0.0.1:9050 diff --git a/doc/translation_process.md b/doc/translation_process.md index 9692832842..b9a10b6527 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -8,7 +8,7 @@ Transifex is setup to monitor the GitHub repo for updates, and when code contain Multiple language support is critical in assisting Bitcoin’s global adoption, and growth. One of Bitcoin’s greatest strengths is cross-border money transfers, any help making that easier is greatly appreciated. -See the [Transifex Bitcoin project](https://www.transifex.com/projects/p/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. +See the [Transifex Bitcoin project](https://www.transifex.com/bitcoin/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. ### Writing code with translations We use automated scripts to help extract translations in both Qt, and non-Qt source files. It is rarely necessary to manually edit the files in `src/qt/locale/`. The translation source files must adhere to the following format: @@ -43,7 +43,7 @@ git commit ### Creating a Transifex account Visit the [Transifex Signup](https://www.transifex.com/signup/) page to create an account. Take note of your username and password, as they will be required to configure the command-line tool. -You can find the Bitcoin translation project at [https://www.transifex.com/projects/p/bitcoin/](https://www.transifex.com/projects/p/bitcoin/). +You can find the Bitcoin translation project at [https://www.transifex.com/bitcoin/bitcoin/](https://www.transifex.com/bitcoin/bitcoin/). ### Installing the Transifex client command-line tool The client is used to fetch updated translations. If you are having problems, or need more details, see [https://docs.transifex.com/client/installing-the-client](https://docs.transifex.com/client/installing-the-client) diff --git a/doc/travis-ci.md b/doc/travis-ci.md deleted file mode 100644 index 38085cec35..0000000000 --- a/doc/travis-ci.md +++ /dev/null @@ -1,42 +0,0 @@ -Travis CI -========= - -Support for using travis-ci has been added in order to automate pull-testing. -See [travis-ci.org](https://travis-ci.org/) for more info - -This procedure is different than the pull-tester that came before it in a few -ways. - -There is nothing to administer. This is a major feature as it means -that builds have no local state. Because there is no ability to login to the -builders to install packages (tools, dependencies, etc), the entire build -procedure must instead be controlled by a declarative script `.travis.yml`. -This script declares each build configuration, creates virtual machines as -necessary, builds, then discards the virtual machines. - -A build matrix is constructed to test a wide range of configurations, rather -than a single pass/fail. This helps to catch build failures and logic errors -that present on platforms other than the ones the author has tested. This -matrix is defined in the build script and can be changed at any time. - -All builders use the dependency-generator in the [depends dir](/depends), rather than -using apt-get to install build dependencies. This guarantees that the tester -is using the same versions as Gitian, so the build results are nearly identical -to what would be found in a final release. However, this also means that builds -will fail if new dependencies are introduced without being added to the -dependency generator. - -In order to avoid rebuilding all dependencies for each build, the binaries are -cached and re-used when possible. Changes in the dependency-generator will -trigger cache-invalidation and rebuilds as necessary. - -These caches can be manually removed if necessary. This is one of the very few -manual operations that is possible with Travis, and it can be done by the -Bitcoin Core committer via the Travis web interface. - -In some cases, secure strings may be needed for hiding sensitive info such as -private keys or URLs. The travis client may be used to create these strings: -http://docs.travis-ci.com/user/encryption-keys/ - -For the details of the build descriptor, see the official docs: -http://docs.travis-ci.com/user/build-configuration/ diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 6062d49c40..b5475dc1c6 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -4,6 +4,10 @@ # Network-related settings: +# Note that if you use testnet or regtest, particularly with the options +# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also +# want to read "[Sections]" further down. + # Run on the test network instead of the real bitcoin network. #testnet=0 @@ -53,6 +57,9 @@ # Listening mode, enabled by default except when 'connect' is being used #listen=1 +# Port on which to listen for connections (default: 8333, testnet: 18333, regtest: 18444) +#port= + # Maximum number of inbound+outbound connections. #maxconnections= @@ -115,6 +122,10 @@ # Wallet options +# Specify where to find wallet, lockfile and logs. If not present, those files will be +# created as new. +#wallet=</path/to/dir> + # Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). # This setting is over-ridden by the -paytxfee option. #txconfirmtarget=n @@ -142,3 +153,19 @@ # Minimize to the system tray #minimizetotray=1 + +# [Sections] +# Most options apply to mainnet, testnet and regtest. +# If you want to confine an option to just one network, you should add it in the +# relevant section below. +# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet +# only apply to mainnet unless they appear in the appropriate section below. + +# Options only for mainnet +[main] + +# Options only for testnet +[test] + +# Options only for regtest +[regtest] diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 046deed9ea..acf9e86667 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -5,7 +5,6 @@ SetCompressor /SOLID lzma # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" -!define VERSION @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@ !define COMPANY "@PACKAGE_NAME@ project" !define URL @PACKAGE_URL@ @@ -49,7 +48,7 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-${VERSION}-win@WINDOWS_BITS@-setup.exe +OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win@WINDOWS_BITS@-setup.exe !if "@WINDOWS_BITS@" == "64" InstallDir $PROGRAMFILES64\Bitcoin !else @@ -59,12 +58,12 @@ CRCCheck on XPStyle on BrandingText " " ShowInstDetails show -VIProductVersion ${VERSION}.@CLIENT_VERSION_BUILD@ +VIProductVersion @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@ VIAddVersionKey ProductName "@PACKAGE_NAME@" -VIAddVersionKey ProductVersion "${VERSION}" +VIAddVersionKey ProductVersion "@PACKAGE_VERSION@" VIAddVersionKey CompanyName "${COMPANY}" VIAddVersionKey CompanyWebsite "${URL}" -VIAddVersionKey FileVersion "${VERSION}" +VIAddVersionKey FileVersion "@PACKAGE_VERSION@" VIAddVersionKey FileDescription "" VIAddVersionKey LegalCopyright "" InstallDirRegKey HKCU "${REGKEY}" Path @@ -99,7 +98,7 @@ Section -post SEC0001 CreateShortcut "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" $INSTDIR\uninstall.exe !insertmacro MUI_STARTMENU_WRITE_END WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayName "$(^Name)" - WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "${VERSION}" + WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "@PACKAGE_VERSION@" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Publisher "${COMPANY}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" URLInfoAbout "${URL}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayIcon $INSTDIR\uninstall.exe diff --git a/src/Makefile.am b/src/Makefile.am index d491530ca1..ed5cab7f04 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -113,7 +113,6 @@ BITCOIN_CORE_H = \ chainparams.h \ chainparamsbase.h \ chainparamsseeds.h \ - checkpoints.h \ checkqueue.h \ clientversion.h \ coins.h \ @@ -124,14 +123,17 @@ BITCOIN_CORE_H = \ compat/sanity.h \ compressor.h \ consensus/consensus.h \ + consensus/tx_check.h \ consensus/tx_verify.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ + flatfile.h \ fs.h \ httprpc.h \ httpserver.h \ index/base.h \ + index/blockfilterindex.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -153,6 +155,8 @@ BITCOIN_CORE_H = \ netaddress.h \ netbase.h \ netmessagemaker.h \ + node/coin.h \ + node/psbt.h \ node/transaction.h \ noui.h \ optional.h \ @@ -161,6 +165,7 @@ BITCOIN_CORE_H = \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ + policy/settings.h \ pow.h \ protocol.h \ psbt.h \ @@ -169,10 +174,9 @@ BITCOIN_CORE_H = \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ - rpc/mining.h \ rpc/protocol.h \ rpc/server.h \ - rpc/rawtransaction.h \ + rpc/rawtransaction_util.h \ rpc/register.h \ rpc/util.h \ scheduler.h \ @@ -199,10 +203,16 @@ BITCOIN_CORE_H = \ undo.h \ util/bip32.h \ util/bytevectorhash.h \ + util/error.h \ + util/fees.h \ util/system.h \ util/memory.h \ util/moneystr.h \ + util/rbf.h \ + util/threadnames.h \ util/time.h \ + util/url.h \ + util/validation.h \ validation.h \ validationinterface.h \ versionbits.h \ @@ -213,6 +223,7 @@ BITCOIN_CORE_H = \ wallet/db.h \ wallet/feebumper.h \ wallet/fees.h \ + wallet/load.h \ wallet/psbtwallet.h \ wallet/rpcwallet.h \ wallet/wallet.h \ @@ -235,37 +246,39 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: shared between bitcoind and bitcoin-qt +# Contains code accessing mempool and chain state that is meant to be separated +# from wallet and gui code (see node/README.md). Shared code should go in +# libbitcoin_common or libbitcoin_util libraries, instead. libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ - bloom.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ - checkpoints.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ + index/blockfilterindex.cpp \ index/txindex.cpp \ interfaces/chain.cpp \ - interfaces/handler.cpp \ interfaces/node.cpp \ init.cpp \ dbwrapper.cpp \ - merkleblock.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ + node/coin.cpp \ + node/psbt.cpp \ node/transaction.cpp \ noui.cpp \ - outputtype.cpp \ policy/fees.cpp \ - policy/policy.cpp \ policy/rbf.cpp \ + policy/settings.cpp \ pow.cpp \ rest.cpp \ rpc/blockchain.cpp \ @@ -274,7 +287,6 @@ libbitcoin_server_a_SOURCES = \ rpc/net.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ - rpc/util.cpp \ script/sigcache.cpp \ shutdown.cpp \ timedata.cpp \ @@ -287,6 +299,9 @@ libbitcoin_server_a_SOURCES = \ versionbits.cpp \ $(BITCOIN_CORE_H) +if ENABLE_WALLET +libbitcoin_server_a_SOURCES += wallet/init.cpp +endif if !ENABLE_WALLET libbitcoin_server_a_SOURCES += dummywallet.cpp endif @@ -313,7 +328,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/db.cpp \ wallet/feebumper.cpp \ wallet/fees.cpp \ - wallet/init.cpp \ + wallet/load.cpp \ wallet/psbtwallet.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ @@ -342,6 +357,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha256.h \ crypto/hmac_sha512.cpp \ crypto/hmac_sha512.h \ + crypto/poly1305.h \ + crypto/poly1305.cpp \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ @@ -385,6 +402,7 @@ libbitcoin_consensus_a_SOURCES = \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ + consensus/tx_check.cpp \ consensus/validation.h \ hash.cpp \ hash.h \ @@ -417,6 +435,7 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ + bloom.cpp \ chainparams.cpp \ coins.cpp \ compressor.cpp \ @@ -425,11 +444,16 @@ libbitcoin_common_a_SOURCES = \ key.cpp \ key_io.cpp \ keystore.cpp \ + merkleblock.cpp \ netaddress.cpp \ netbase.cpp \ + outputtype.cpp \ policy/feerate.cpp \ - psbt.cpp \ + policy/policy.cpp \ protocol.cpp \ + psbt.cpp \ + rpc/rawtransaction_util.cpp \ + rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ script/ismine.cpp \ @@ -452,6 +476,7 @@ libbitcoin_util_a_SOURCES = \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ fs.cpp \ + interfaces/handler.cpp \ logging.cpp \ random.cpp \ rpc/protocol.cpp \ @@ -460,10 +485,16 @@ libbitcoin_util_a_SOURCES = \ threadinterrupt.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/error.cpp \ + util/fees.cpp \ util/system.cpp \ util/moneystr.cpp \ + util/rbf.cpp \ + util/threadnames.cpp \ util/strencodings.cpp \ util/time.cpp \ + util/url.cpp \ + util/validation.cpp \ $(BITCOIN_CORE_H) if GLIBC_BACK_COMPAT @@ -491,6 +522,8 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif +# Libraries below may be listed more than once to resolve circular dependencies (see +# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ @@ -561,17 +594,18 @@ endif bitcoin_wallet_LDADD = \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_ZMQ) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ - $(LIBSECP256K1) + $(LIBSECP256K1) \ + $(LIBUNIVALUE) -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # @@ -638,13 +672,13 @@ clean-local: check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) endif if ENABLE_BIP70 diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 5e787ca222..ae7eb19ceb 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -26,17 +26,24 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/rpc_mempool.cpp \ bench/verify_script.cpp \ bench/base58.cpp \ bench/bech32.cpp \ bench/lockedpool.cpp \ - bench/prevector.cpp + bench/poly1305.cpp \ + bench/prevector.cpp \ + test/setup_common.h \ + test/setup_common.cpp \ + test/util.h \ + test/util.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ @@ -47,7 +54,9 @@ bench_bench_bitcoin_LDADD = \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ - $(LIBUNIVALUE) + $(LIBUNIVALUE) \ + $(EVENT_PTHREADS_LIBS) \ + $(EVENT_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -55,9 +64,10 @@ endif if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 61977b50cd..4acfff809e 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -33,10 +33,10 @@ TEST_QT_H = \ qt/test/wallettests.h TEST_BITCOIN_CPP = \ - test/test_bitcoin.cpp + test/setup_common.cpp TEST_BITCOIN_H = \ - test/test_bitcoin.h + test/setup_common.h qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 84bc326cfe..1144ca8a78 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -51,14 +51,32 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ - test/test_bitcoin_main.cpp \ - test/test_bitcoin.h \ - test/test_bitcoin.cpp + test/main.cpp \ + test/setup_common.h \ + test/setup_common.cpp FUZZ_SUITE = \ + test/setup_common.h \ + test/setup_common.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h +FUZZ_SUITE_LD_COMMON = \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBUNIVALUE) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(BOOST_LIBS) \ + $(LIBMEMENV) \ + $(LIBSECP256K1) \ + $(EVENT_LIBS) \ + $(CRYPTO_LIBS) \ + $(EVENT_PTHREADS_LIBS) + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -74,6 +92,7 @@ BITCOIN_TESTS =\ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockfilter_index_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -83,6 +102,7 @@ BITCOIN_TESTS =\ test/cuckoocache_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ @@ -90,7 +110,7 @@ BITCOIN_TESTS =\ test/key_tests.cpp \ test/limitedmap_tests.cpp \ test/dbwrapper_tests.cpp \ - test/main_tests.cpp \ + test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ @@ -118,6 +138,7 @@ BITCOIN_TESTS =\ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ + test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ @@ -169,7 +190,7 @@ test_test_bitcoin_LDADD += $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDC test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ -test_test_bitcoin_LDADD += $(ZMQ_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ @@ -177,358 +198,127 @@ test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_block_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTRANSACTION_DESERIALIZE=1 test_fuzz_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_transaction_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_transaction_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocklocator_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocklocator_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockmerkleroot_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockmerkleroot_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_addrman_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_addrman_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_banentry_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_banentry_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_coins_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_coins_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_netaddr_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_netaddr_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_script_flags_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_script_flags_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_service_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_service_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_messageheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_messageheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_address_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_address_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_inv_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_inv_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_bloomfilter_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_bloomfilter_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_diskblockindex_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_diskblockindex_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txoutcompressor_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txoutcompressor_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactions_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactionsrequest_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) endif # ENABLE_FUZZ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrman.cpp b/src/addrman.cpp index 06c342ba73..8a5f78d1c5 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -239,7 +239,9 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime // Will moving this address into tried evict another entry? if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table, moving %s to m_tried_collisions=%d\n", addr.ToString(), m_tried_collisions.size()); + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { m_tried_collisions.insert(nId); } @@ -561,12 +563,19 @@ void CAddrMan::ResolveCollisions_() // Give address at least 60 seconds to successfully connect if (GetAdjustedTime() - info_old.nLastTry > 60) { - LogPrint(BCLog::ADDRMAN, "Swapping %s for %s in tried table\n", info_new.ToString(), info_old.ToString()); + LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); // Replaces an existing address already in the tried table with the new address Good_(info_new, false, GetAdjustedTime()); erase_collision = true; } + } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + // If the collision hasn't resolved in some reasonable amount of time, + // just evict the old entry -- we must not be able to + // connect to it for some reason. + LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; } } else { // Collision is not actually a collision anymore Good_(info_new, false, GetAdjustedTime()); diff --git a/src/addrman.h b/src/addrman.h index 003bd059f8..e54184ce35 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -166,6 +166,9 @@ public: //! the maximum number of tried addr collisions to store #define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 +//! the maximum time we'll spend trying to resolve a tried table collision, in seconds +static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes + /** * Stochastical (IP) address manager */ diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 966b99f6c8..b08ecbb621 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,15 +1,19 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> +#include <chainparams.h> +#include <test/setup_common.h> +#include <validation.h> + +#include <algorithm> #include <assert.h> -#include <iostream> #include <iomanip> -#include <algorithm> -#include <regex> +#include <iostream> #include <numeric> +#include <regex> void benchmark::ConsolePrinter::header() { @@ -108,6 +112,13 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { + TestingSetup test{CBaseChainParams::REGTEST}; + { + assert(::chainActive.Height() == 0); + const bool witness_enabled{IsWitnessEnabled(::chainActive.Tip(), Params().GetConsensus())}; + assert(witness_enabled); + } + if (!std::regex_match(p.first, baseMatch, reFilter)) { continue; } diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index d67b2c5bc7..3cf0bf9530 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,14 +6,11 @@ #include <crypto/sha256.h> #include <key.h> -#include <util/system.h> #include <util/strencodings.h> -#include <validation.h> +#include <util/system.h> #include <memory> -const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; - static const int64_t DEFAULT_BENCH_EVALUATIONS = 5; static const char* DEFAULT_BENCH_FILTER = ".*"; static const char* DEFAULT_BENCH_SCALING = "1.0"; @@ -36,14 +33,6 @@ static void SetupBenchArgs() gArgs.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), false, OptionsCategory::OPTIONS); } -static fs::path SetDataDir() -{ - fs::path ret = fs::temp_directory_path() / "bench_bitcoin" / fs::unique_path(); - fs::create_directories(ret); - gArgs.ForceSetArg("-datadir", ret.string()); - return ret; -} - int main(int argc, char** argv) { SetupBenchArgs(); @@ -59,13 +48,6 @@ int main(int argc, char** argv) return EXIT_SUCCESS; } - // Set the datadir after parsing the bench options - const fs::path bench_datadir{SetDataDir()}; - - SHA256AutoDetect(); - ECC_Start(); - SetupEnvironment(); - int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); @@ -88,9 +70,5 @@ int main(int argc, char** argv) benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); - fs::remove_all(bench_datadir); - - ECC_Stop(); - return EXIT_SUCCESS; } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index cd4543c012..fb33c09ab2 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,58 +1,18 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <chainparams.h> -#include <coins.h> -#include <consensus/merkle.h> #include <consensus/validation.h> #include <crypto/sha256.h> -#include <miner.h> -#include <policy/policy.h> -#include <pow.h> -#include <scheduler.h> -#include <txdb.h> +#include <test/util.h> #include <txmempool.h> -#include <util/time.h> #include <validation.h> -#include <validationinterface.h> -#include <boost/thread.hpp> #include <list> #include <vector> -static std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey) -{ - auto block = std::make_shared<CBlock>( - BlockAssembler{Params()} - .CreateNewBlock(coinbase_scriptPubKey) - ->block); - - block->nTime = ::chainActive.Tip()->GetMedianTimePast() + 1; - block->hashMerkleRoot = BlockMerkleRoot(*block); - - return block; -} - - -static CTxIn MineBlock(const CScript& coinbase_scriptPubKey) -{ - auto block = PrepareBlock(coinbase_scriptPubKey); - - while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { - ++block->nNonce; - assert(block->nNonce); - } - - bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; - assert(processed); - - return CTxIn{block->vtx[0]->GetHash(), 0}; -} - - static void AssembleBlock(benchmark::State& state) { const std::vector<unsigned char> op_true{OP_TRUE}; @@ -64,32 +24,6 @@ static void AssembleBlock(benchmark::State& state) const CScript SCRIPT_PUB{CScript(OP_0) << std::vector<unsigned char>{witness_program.begin(), witness_program.end()}}; - // Switch to regtest so we can mine faster - // Also segwit is active, so we can include witness transactions - SelectParams(CBaseChainParams::REGTEST); - - InitScriptExecutionCache(); - - boost::thread_group thread_group; - CScheduler scheduler; - { - LOCK(cs_main); - ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); - ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - } - { - const CChainParams& chainparams = Params(); - thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); - GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - LoadGenesisBlock(chainparams); - CValidationState state; - ActivateBestChain(state, chainparams); - assert(::chainActive.Tip() != nullptr); - const bool witness_enabled{IsWitnessEnabled(::chainActive.Tip(), chainparams.GetConsensus())}; - assert(witness_enabled); - } - // Collect some loose transactions that spend the coinbases of our mined blocks constexpr size_t NUM_BLOCKS{200}; std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; @@ -114,11 +48,6 @@ static void AssembleBlock(benchmark::State& state) while (state.KeepRunning()) { PrepareBlock(SCRIPT_PUB); } - - thread_group.interrupt_all(); - thread_group.join_all(); - GetMainSignals().FlushBackgroundCallbacks(); - GetMainSignals().UnregisterBackgroundSignalScheduler(); } BENCHMARK(AssembleBlock, 700); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 74641191a1..f2ab03e20e 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -29,7 +29,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<st static void CoinSelection(benchmark::State& state) { auto chain = interfaces::MakeChain(); - const CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + const CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -61,7 +61,7 @@ static void CoinSelection(benchmark::State& state) typedef std::set<CInputCoin> CoinSet; static auto testChain = interfaces::MakeChain(); -static const CWallet testWallet(*testChain, WalletLocation(), WalletDatabase::CreateDummy()); +static const CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 1f6840d813..2d7a351523 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,15 +10,11 @@ #include <miner.h> #include <policy/policy.h> #include <pow.h> -#include <scheduler.h> -#include <txdb.h> +#include <test/util.h> #include <txmempool.h> -#include <util/time.h> #include <validation.h> #include <validationinterface.h> -#include <boost/thread.hpp> - #include <list> #include <vector> @@ -27,31 +23,7 @@ static void DuplicateInputs(benchmark::State& state) { const CScript SCRIPT_PUB{CScript(OP_TRUE)}; - // Switch to regtest so we can mine faster - // Also segwit is active, so we can include witness transactions - SelectParams(CBaseChainParams::REGTEST); - - InitScriptExecutionCache(); - - boost::thread_group thread_group; - CScheduler scheduler; const CChainParams& chainparams = Params(); - { - LOCK(cs_main); - ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); - ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - } - { - thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); - GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - LoadGenesisBlock(chainparams); - CValidationState cvstate; - ActivateBestChain(cvstate, chainparams); - assert(::chainActive.Tip() != nullptr); - const bool witness_enabled{IsWitnessEnabled(::chainActive.Tip(), chainparams.GetConsensus())}; - assert(witness_enabled); - } CBlock block{}; CMutableTransaction coinbaseTx{}; @@ -92,11 +64,6 @@ static void DuplicateInputs(benchmark::State& state) assert(!CheckBlock(block, cvstate, chainparams.GetConsensus(), false, false)); assert(cvstate.GetRejectReason() == "bad-txns-inputs-duplicate"); } - - thread_group.interrupt_all(); - thread_group.join_all(); - GetMainSignals().FlushBackgroundCallbacks(); - GetMainSignals().UnregisterBackgroundSignalScheduler(); } BENCHMARK(DuplicateInputs, 10); diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp new file mode 100644 index 0000000000..16342d0fbe --- /dev/null +++ b/src/bench/poly1305.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <iostream> + +#include <bench/bench.h> +#include <crypto/poly1305.h> + +/* Number of bytes to process per iteration */ +static constexpr uint64_t BUFFER_SIZE_TINY = 64; +static constexpr uint64_t BUFFER_SIZE_SMALL = 256; +static constexpr uint64_t BUFFER_SIZE_LARGE = 1024*1024; + +static void POLY1305(benchmark::State& state, size_t buffersize) +{ + std::vector<unsigned char> tag(POLY1305_TAGLEN, 0); + std::vector<unsigned char> key(POLY1305_KEYLEN, 0); + std::vector<unsigned char> in(buffersize, 0); + while (state.KeepRunning()) + poly1305_auth(tag.data(), in.data(), in.size(), key.data()); +} + +static void POLY1305_64BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_TINY); +} + +static void POLY1305_256BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_SMALL); +} + +static void POLY1305_1MB(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_LARGE); +} + +BENCHMARK(POLY1305_64BYTES, 500000); +BENCHMARK(POLY1305_256BYTES, 250000); +BENCHMARK(POLY1305_1MB, 340); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp new file mode 100644 index 0000000000..67d8a25564 --- /dev/null +++ b/src/bench/rpc_mempool.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <rpc/blockchain.h> +#include <txmempool.h> + +#include <univalue.h> + +#include <list> +#include <vector> + +static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp)); +} + +static void RpcMempool(benchmark::State& state) +{ + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + + for (int i = 0; i < 1000; ++i) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vin[0].scriptWitness.stack.push_back({1}); + tx.vout.resize(1); + tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx.vout[0].nValue = i; + const CTransactionRef tx_r{MakeTransactionRef(tx)}; + AddTx(tx_r, /* fee */ i, pool); + } + + while (state.KeepRunning()) { + (void)MempoolToJSON(pool, /*verbose*/ true); + } +} + +BENCHMARK(RpcMempool, 40); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp new file mode 100644 index 0000000000..46ca12826b --- /dev/null +++ b/src/bench/wallet_balance.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2012-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <interfaces/chain.h> +#include <key_io.h> +#include <optional.h> +#include <test/util.h> +#include <validationinterface.h> +#include <wallet/wallet.h> + +static void WalletBalance(benchmark::State& state, const bool set_dirty, const bool add_watchonly, const bool add_mine) +{ + const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; + + std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(); + CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; + { + bool first_run; + if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); + wallet.handleNotifications(); + } + + + const Optional<std::string> address_mine{add_mine ? Optional<std::string>{getnewaddress(wallet)} : nullopt}; + if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); + + for (int i = 0; i < 100; ++i) { + generatetoaddress(address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(ADDRESS_WATCHONLY); + } + SyncWithValidationInterfaceQueue(); + + auto bal = wallet.GetBalance(); // Cache + + while (state.KeepRunning()) { + if (set_dirty) wallet.MarkDirty(); + bal = wallet.GetBalance(); + if (add_mine) assert(bal.m_mine_trusted > 0); + if (add_watchonly) assert(bal.m_watchonly_trusted > 0); + } +} + +static void WalletBalanceDirty(benchmark::State& state) { WalletBalance(state, /* set_dirty */ true, /* add_watchonly */ true, /* add_mine */ true); } +static void WalletBalanceClean(benchmark::State& state) { WalletBalance(state, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ true); } +static void WalletBalanceMine(benchmark::State& state) { WalletBalance(state, /* set_dirty */ false, /* add_watchonly */ false, /* add_mine */ true); } +static void WalletBalanceWatch(benchmark::State& state) { WalletBalance(state, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ false); } + +BENCHMARK(WalletBalanceDirty, 2500); +BENCHMARK(WalletBalanceClean, 8000); +BENCHMARK(WalletBalanceMine, 16000); +BENCHMARK(WalletBalanceWatch, 8000); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index b0e1f67d93..1009a771f8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -253,16 +253,12 @@ public: } result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); - if (!batch[ID_WALLETINFO].isNull()) { - result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); - } result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); - result.pushKV("testnet", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"].get_str() == "test")); + result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO].isNull()) { result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 4be89aab6c..7f41ea7aed 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -18,6 +18,7 @@ #include <script/script.h> #include <script/sign.h> #include <univalue.h> +#include <util/rbf.h> #include <util/system.h> #include <util/moneystr.h> #include <util/strencodings.h> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index dde75c1b12..b31f86cdd9 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -19,6 +19,7 @@ #include <util/system.h> #include <httpserver.h> #include <httprpc.h> +#include <util/threadnames.h> #include <util/strencodings.h> #include <walletinitinterface.h> @@ -64,6 +65,8 @@ static bool AppInit(int argc, char* argv[]) bool fRet = false; + util::ThreadRename("init"); + // // Parameters // diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 10f51931f0..f0fcf675eb 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -203,7 +203,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< // but that is expensive, and CheckBlock caches a block's // "checked-status" (in the CBlock?). CBlock should be able to // check its own merkle root and cache that check. - if (state.CorruptionPossible()) + if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) return READ_STATUS_FAILED; // Possible Short ID collision return READ_STATUS_CHECKBLOCK_FAILED; } diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index bcf24047ff..787390be31 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -2,6 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <mutex> +#include <sstream> + #include <blockfilter.h> #include <crypto/siphash.h> #include <hash.h> @@ -15,6 +18,10 @@ static constexpr int GCS_SER_TYPE = SER_NETWORK; /// Protocol version used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_VERSION = 0; +static const std::map<BlockFilterType, std::string> g_filter_types = { + {BlockFilterType::BASIC, "basic"}, +}; + template <typename OStream> static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) { @@ -197,6 +204,57 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const return MatchInternal(queries.data(), queries.size()); } +const std::string& BlockFilterTypeName(BlockFilterType filter_type) +{ + static std::string unknown_retval = ""; + auto it = g_filter_types.find(filter_type); + return it != g_filter_types.end() ? it->second : unknown_retval; +} + +bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) { + for (const auto& entry : g_filter_types) { + if (entry.second == name) { + filter_type = entry.first; + return true; + } + } + return false; +} + +const std::vector<BlockFilterType>& AllBlockFilterTypes() +{ + static std::vector<BlockFilterType> types; + + static std::once_flag flag; + std::call_once(flag, []() { + types.reserve(g_filter_types.size()); + for (auto entry : g_filter_types) { + types.push_back(entry.first); + } + }); + + return types; +} + +const std::string& ListBlockFilterTypes() +{ + static std::string type_list; + + static std::once_flag flag; + std::call_once(flag, []() { + std::stringstream ret; + bool first = true; + for (auto entry : g_filter_types) { + if (!first) ret << ", "; + ret << entry.second; + first = false; + } + type_list = ret.str(); + }); + + return type_list; +} + static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, const CBlockUndo& block_undo) { @@ -251,6 +309,8 @@ bool BlockFilter::BuildParams(GCSFilter::Params& params) const params.m_P = BASIC_FILTER_P; params.m_M = BASIC_FILTER_M; return true; + case BlockFilterType::INVALID: + return false; } return false; diff --git a/src/blockfilter.h b/src/blockfilter.h index 4d1f51dd60..914b94fec1 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -6,6 +6,7 @@ #define BITCOIN_BLOCKFILTER_H #include <stdint.h> +#include <string> #include <unordered_set> #include <vector> @@ -83,11 +84,24 @@ public: constexpr uint8_t BASIC_FILTER_P = 19; constexpr uint32_t BASIC_FILTER_M = 784931; -enum BlockFilterType : uint8_t +enum class BlockFilterType : uint8_t { BASIC = 0, + INVALID = 255, }; +/** Get the human-readable name for a filter type. Returns empty string for unknown types. */ +const std::string& BlockFilterTypeName(BlockFilterType filter_type); + +/** Find a filter type by its human-readable name. */ +bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type); + +/** Get a list of known filter types. */ +const std::vector<BlockFilterType>& AllBlockFilterTypes(); + +/** Get a comma-separated list of known filter type names. */ +const std::string& ListBlockFilterTypes(); + /** * Complete block filter struct as defined in BIP 157. Serialization matches * payload of "cfilter" messages. @@ -95,7 +109,7 @@ enum BlockFilterType : uint8_t class BlockFilter { private: - BlockFilterType m_filter_type; + BlockFilterType m_filter_type = BlockFilterType::INVALID; uint256 m_block_hash; GCSFilter m_filter; diff --git a/src/chain.cpp b/src/chain.cpp index d462f94ab5..5520d8149a 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -59,10 +59,11 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const { return pindex; } -CBlockIndex* CChain::FindEarliestAtLeast(int64_t nTime) const +CBlockIndex* CChain::FindEarliestAtLeast(int64_t nTime, int height) const { - std::vector<CBlockIndex*>::const_iterator lower = std::lower_bound(vChain.begin(), vChain.end(), nTime, - [](CBlockIndex* pBlock, const int64_t& time) -> bool { return pBlock->GetBlockTimeMax() < time; }); + std::pair<int64_t, int> blockparams = std::make_pair(nTime, height); + std::vector<CBlockIndex*>::const_iterator lower = std::lower_bound(vChain.begin(), vChain.end(), blockparams, + [](CBlockIndex* pBlock, const std::pair<int64_t, int>& blockparams) -> bool { return pBlock->GetBlockTimeMax() < blockparams.first || pBlock->nHeight < blockparams.second; }); return (lower == vChain.end() ? nullptr : *lower); } diff --git a/src/chain.h b/src/chain.h index 5a6f10b84f..dd9cc2a598 100644 --- a/src/chain.h +++ b/src/chain.h @@ -8,6 +8,7 @@ #include <arith_uint256.h> #include <consensus/params.h> +#include <flatfile.h> #include <primitives/block.h> #include <tinyformat.h> #include <uint256.h> @@ -90,46 +91,6 @@ public: } }; -struct CDiskBlockPos -{ - int nFile; - unsigned int nPos; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } - - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return (a.nFile == b.nFile && a.nPos == b.nPos); - } - - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return !(a == b); - } - - void SetNull() { nFile = -1; nPos = 0; } - bool IsNull() const { return (nFile == -1); } - - std::string ToString() const - { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); - } - -}; - enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -266,8 +227,8 @@ public: nNonce = block.nNonce; } - CDiskBlockPos GetBlockPos() const { - CDiskBlockPos ret; + FlatFilePos GetBlockPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; ret.nPos = nDataPos; @@ -275,8 +236,8 @@ public: return ret; } - CDiskBlockPos GetUndoPos() const { - CDiskBlockPos ret; + FlatFilePos GetUndoPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; ret.nPos = nUndoPos; @@ -504,8 +465,8 @@ public: /** Find the last common block between this chain and a block index entry. */ const CBlockIndex *FindFork(const CBlockIndex *pindex) const; - /** Find the earliest block with timestamp equal or greater than the given. */ - CBlockIndex* FindEarliestAtLeast(int64_t nTime) const; + /** Find the earliest block with timestamp equal or greater than the given time and height equal or greater than the given height. */ + CBlockIndex* FindEarliestAtLeast(int64_t nTime, int height) const; }; #endif // BITCOIN_CHAIN_H diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp deleted file mode 100644 index ad5edfeb39..0000000000 --- a/src/checkpoints.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <checkpoints.h> - -#include <chain.h> -#include <chainparams.h> -#include <reverse_iterator.h> -#include <validation.h> - -#include <stdint.h> - - -namespace Checkpoints { - - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) - { - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) - { - const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); - if (pindex) { - return pindex; - } - } - return nullptr; - } - -} // namespace Checkpoints diff --git a/src/checkpoints.h b/src/checkpoints.h deleted file mode 100644 index a25e97e469..0000000000 --- a/src/checkpoints.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_CHECKPOINTS_H -#define BITCOIN_CHECKPOINTS_H - -#include <uint256.h> - -#include <map> - -class CBlockIndex; -struct CCheckpointData; - -/** - * Block-chain checkpoints are compiled-in sanity checks. - * They are updated every release or three. - */ -namespace Checkpoints -{ - -//! Returns last CBlockIndex* that is a checkpoint -CBlockIndex* GetLastCheckpoint(const CCheckpointData& data); - -} //namespace Checkpoints - -#endif // BITCOIN_CHECKPOINTS_H diff --git a/src/coins.h b/src/coins.h index d39ebf9062..482e233e8c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -294,6 +294,10 @@ public: bool HaveInputs(const CTransaction& tx) const; private: + /** + * @note this is marked const, but may actually append to `cacheCoins`, increasing + * memory usage. + */ CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; }; diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 820c9b93d9..6e7b4d3ded 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -17,6 +17,17 @@ # error "Bitcoin cannot be compiled without assertions." #endif +// Assumption: We assume a C++11 (ISO/IEC 14882:2011) compiler (minimum requirement). +// Example(s): We assume the presence of C++11 features everywhere :-) +// Note: MSVC does not report the expected __cplusplus value due to legacy +// reasons. +#if !defined(_MSC_VER) +// ISO Standard C++11 [cpp.predefined]p1: +// "The name __cplusplus is defined to the value 201103L when compiling a C++ +// translation unit." +static_assert(__cplusplus >= 201103L, "C++11 standard assumed"); +#endif + // Assumption: We assume the floating-point types to fulfill the requirements of // IEC 559 (IEEE 754) standard. // Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction @@ -40,8 +51,13 @@ static_assert(sizeof(double) == 8, "64-bit double assumed"); static_assert(sizeof(short) == 2, "16-bit short assumed"); static_assert(sizeof(int) == 4, "32-bit int assumed"); +// Assumption: We assume size_t to be 32-bit or 64-bit. +// Example(s): size_t assumed to be at least 32-bit in ecdsa_signature_parse_der_lax(...). +// size_t assumed to be 32-bit or 64-bit in MallocUsage(...). +static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, "size_t assumed to be 32-bit or 64-bit"); +static_assert(sizeof(size_t) == sizeof(void*), "Sizes of size_t and void* assumed to be equal"); + // Some important things we are NOT assuming (non-exhaustive list): -// * We are NOT assuming a specific value for sizeof(std::size_t). // * We are NOT assuming a specific value for std::endian::native. // * We are NOT assuming a specific value for std::locale("").name(). // * We are NOT assuming a specific value for std::numeric_limits<char>::is_signed. diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp new file mode 100644 index 0000000000..23ed3ecb53 --- /dev/null +++ b/src/consensus/tx_check.cpp @@ -0,0 +1,57 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/tx_check.h> + +#include <primitives/transaction.h> +#include <consensus/validation.h> + +bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) +{ + // Basic checks that don't depend on any context + if (tx.vin.empty()) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vin-empty"); + if (tx.vout.empty()) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-empty"); + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-oversize"); + + // Check for negative or overflow output values + CAmount nValueOut = 0; + for (const auto& txout : tx.vout) + { + if (txout.nValue < 0) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-negative"); + if (txout.nValue > MAX_MONEY) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-toolarge"); + nValueOut += txout.nValue; + if (!MoneyRange(nValueOut)) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + } + + // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock + if (fCheckDuplicateInputs) { + std::set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) + { + if (!vInOutPoints.insert(txin.prevout).second) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + } + } + + if (tx.IsCoinBase()) + { + if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-length"); + } + else + { + for (const auto& txin : tx.vin) + if (txin.prevout.IsNull()) + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-prevout-null"); + } + + return true; +} diff --git a/src/consensus/tx_check.h b/src/consensus/tx_check.h new file mode 100644 index 0000000000..bcfdf36bf9 --- /dev/null +++ b/src/consensus/tx_check.h @@ -0,0 +1,20 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONSENSUS_TX_CHECK_H +#define BITCOIN_CONSENSUS_TX_CHECK_H + +/** + * Context-independent transaction checking code that can be called outside the + * bitcoin server and doesn't depend on chain or mempool state. Transaction + * verification code that does call server functions or depend on server state + * belongs in tx_verify.h/cpp instead. + */ + +class CTransaction; +class CValidationState; + +bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); + +#endif // BITCOIN_CONSENSUS_TX_CHECK_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 0a7eacfb91..4b93cae848 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -156,60 +156,11 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i return nSigOps; } -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) -{ - // Basic checks that don't depend on any context - if (tx.vin.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); - if (tx.vout.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); - - // Check for negative or overflow output values - CAmount nValueOut = 0; - for (const auto& txout : tx.vout) - { - if (txout.nValue < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); - if (txout.nValue > MAX_MONEY) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); - nValueOut += txout.nValue; - if (!MoneyRange(nValueOut)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); - } - - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - } - } - - if (tx.IsCoinBase()) - { - if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); - } - else - { - for (const auto& txin : tx.vin) - if (txin.prevout.IsNull()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); - } - - return true; -} - bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) { // are the actual inputs available? if (!inputs.HaveInputs(tx)) { - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-missingorspent", false, + return state.Invalid(ValidationInvalidReason::TX_MISSING_INPUTS, false, REJECT_INVALID, "bad-txns-inputs-missingorspent", strprintf("%s: inputs missing/spent", __func__)); } @@ -221,28 +172,27 @@ bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, c // If prev is coinbase, check that it's matured if (coin.IsCoinBase() && nSpendHeight - coin.nHeight < COINBASE_MATURITY) { - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values nValueIn += coin.out.nValue; if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) { - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } } const CAmount value_out = tx.GetValueOut(); if (nValueIn < value_out) { - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-in-belowout", strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(value_out))); } // Tally transaction fees const CAmount txfee_aux = nValueIn - value_out; if (!MoneyRange(txfee_aux)) { - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-fee-outofrange"); } txfee = txfee_aux; diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 0519cef8c0..3519fc555d 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -17,9 +17,6 @@ class CValidationState; /** Transaction validation functions */ -/** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); - namespace Consensus { /** * Check whether all inputs of this transaction are valid (no double spends and amounts) diff --git a/src/consensus/validation.h b/src/consensus/validation.h index f2e2c3585a..2e23f4b3a4 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -22,6 +22,78 @@ static const unsigned char REJECT_NONSTANDARD = 0x40; static const unsigned char REJECT_INSUFFICIENTFEE = 0x42; static const unsigned char REJECT_CHECKPOINT = 0x43; +/** A "reason" why something was invalid, suitable for determining whether the + * provider of the object should be banned/ignored/disconnected/etc. + * These are much more granular than the rejection codes, which may be more + * useful for some other use-cases. + */ +enum class ValidationInvalidReason { + // txn and blocks: + NONE, //!< not actually invalid + CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) + /** + * Invalid by a change to consensus rules more recent than SegWit. + * Currently unused as there are no such consensus rule changes, and any download + * sources realistically need to support SegWit in order to provide useful data, + * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork + * is uninteresting. + */ + RECENT_CONSENSUS_CHANGE, + // Only blocks (or headers): + CACHED_INVALID, //!< this object was cached as being invalid, but we don't know why + BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old + BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW + BLOCK_MISSING_PREV, //!< We don't have the previous block the checked one is built on + BLOCK_INVALID_PREV, //!< A block this one builds on is invalid + BLOCK_TIME_FUTURE, //!< block timestamp was > 2 hours in the future (or our clock is bad) + BLOCK_CHECKPOINT, //!< the block failed to meet one of our checkpoints + // Only loose txn: + TX_NOT_STANDARD, //!< didn't meet our local policy rules + TX_MISSING_INPUTS, //!< a transaction was missing some of its inputs + TX_PREMATURE_SPEND, //!< transaction spends a coinbase too early, or violates locktime/sequence locks + /** + * Transaction might be missing a witness, have a witness prior to SegWit + * activation, or witness may have been malleated (which includes + * non-standard witnesses). + */ + TX_WITNESS_MUTATED, + /** + * Tx already in mempool or conflicts with a tx in the chain + * (if it conflicts with another tx in mempool, we use MEMPOOL_POLICY as it failed to reach the RBF threshold) + * TODO: Currently this is only used if the transaction already exists in the mempool or on chain, + * TODO: ATMP's fMissingInputs and a valid CValidationState being used to indicate missing inputs + */ + TX_CONFLICT, + TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits +}; + +inline bool IsTransactionReason(ValidationInvalidReason r) +{ + return r == ValidationInvalidReason::NONE || + r == ValidationInvalidReason::CONSENSUS || + r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || + r == ValidationInvalidReason::TX_NOT_STANDARD || + r == ValidationInvalidReason::TX_PREMATURE_SPEND || + r == ValidationInvalidReason::TX_MISSING_INPUTS || + r == ValidationInvalidReason::TX_WITNESS_MUTATED || + r == ValidationInvalidReason::TX_CONFLICT || + r == ValidationInvalidReason::TX_MEMPOOL_POLICY; +} + +inline bool IsBlockReason(ValidationInvalidReason r) +{ + return r == ValidationInvalidReason::NONE || + r == ValidationInvalidReason::CONSENSUS || + r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || + r == ValidationInvalidReason::CACHED_INVALID || + r == ValidationInvalidReason::BLOCK_INVALID_HEADER || + r == ValidationInvalidReason::BLOCK_MUTATED || + r == ValidationInvalidReason::BLOCK_MISSING_PREV || + r == ValidationInvalidReason::BLOCK_INVALID_PREV || + r == ValidationInvalidReason::BLOCK_TIME_FUTURE || + r == ValidationInvalidReason::BLOCK_CHECKPOINT; +} + /** Capture information about block/transaction validation */ class CValidationState { private: @@ -30,32 +102,24 @@ private: MODE_INVALID, //!< network rule violation (DoS value may be set) MODE_ERROR, //!< run-time error } mode; - int nDoS; + ValidationInvalidReason m_reason; std::string strRejectReason; unsigned int chRejectCode; - bool corruptionPossible; std::string strDebugMessage; public: - CValidationState() : mode(MODE_VALID), nDoS(0), chRejectCode(0), corruptionPossible(false) {} - bool DoS(int level, bool ret = false, - unsigned int chRejectCodeIn=0, const std::string &strRejectReasonIn="", - bool corruptionIn=false, - const std::string &strDebugMessageIn="") { + CValidationState() : mode(MODE_VALID), m_reason(ValidationInvalidReason::NONE), chRejectCode(0) {} + bool Invalid(ValidationInvalidReason reasonIn, bool ret = false, + unsigned int chRejectCodeIn=0, const std::string &strRejectReasonIn="", + const std::string &strDebugMessageIn="") { + m_reason = reasonIn; chRejectCode = chRejectCodeIn; strRejectReason = strRejectReasonIn; - corruptionPossible = corruptionIn; strDebugMessage = strDebugMessageIn; if (mode == MODE_ERROR) return ret; - nDoS += level; mode = MODE_INVALID; return ret; } - bool Invalid(bool ret = false, - unsigned int _chRejectCode=0, const std::string &_strRejectReason="", - const std::string &_strDebugMessage="") { - return DoS(0, ret, _chRejectCode, _strRejectReason, false, _strDebugMessage); - } bool Error(const std::string& strRejectReasonIn) { if (mode == MODE_VALID) strRejectReason = strRejectReasonIn; @@ -71,19 +135,7 @@ public: bool IsError() const { return mode == MODE_ERROR; } - bool IsInvalid(int &nDoSOut) const { - if (IsInvalid()) { - nDoSOut = nDoS; - return true; - } - return false; - } - bool CorruptionPossible() const { - return corruptionPossible; - } - void SetCorruptionPossible() { - corruptionPossible = true; - } + ValidationInvalidReason GetReason() const { return m_reason; } unsigned int GetRejectCode() const { return chRejectCode; } std::string GetRejectReason() const { return strRejectReason; } std::string GetDebugMessage() const { return strDebugMessage; } diff --git a/src/core_io.h b/src/core_io.h index ae377eb6e8..19fb7b29f6 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -16,7 +16,6 @@ class CBlockHeader; class CScript; class CTransaction; struct CMutableTransaction; -struct PartiallySignedTransaction; class uint256; class UniValue; @@ -37,11 +36,6 @@ bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); */ bool ParseHashStr(const std::string& strHex, uint256& result); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); - -//! Decode a base64ed PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); -//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); int ParseSighashString(const UniValue& sighash); // core_write.cpp diff --git a/src/core_read.cpp b/src/core_read.cpp index 536a7f4f17..a879a375ce 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -4,7 +4,6 @@ #include <core_io.h> -#include <psbt.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/script.h> @@ -177,33 +176,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) -{ - bool invalid; - std::string tx_data = DecodeBase64(base64_tx, &invalid); - if (invalid) { - error = "invalid base64"; - return false; - } - return DecodeRawPSBT(psbt, tx_data, error); -} - -bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) -{ - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); - try { - ss_data >> psbt; - if (!ss_data.empty()) { - error = "extra data after PSBT"; - return false; - } - } catch (const std::exception& e) { - error = e.what(); - return false; - } - return true; -} - bool ParseHashStr(const std::string& strHex, uint256& result) { if ((strHex.size() != 64) || !IsHex(strHex)) diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 919ea593b7..2dc2133434 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -12,36 +12,6 @@ extern "C" { #include <crypto/ctaes/ctaes.c> } -AES128Encrypt::AES128Encrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Encrypt::~AES128Encrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Encrypt::Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const -{ - AES128_encrypt(&ctx, 1, ciphertext, plaintext); -} - -AES128Decrypt::AES128Decrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Decrypt::~AES128Decrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Decrypt::Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const -{ - AES128_decrypt(&ctx, 1, plaintext, ciphertext); -} - AES256Encrypt::AES256Encrypt(const unsigned char key[32]) { AES256_init(&ctx, key); @@ -182,35 +152,3 @@ AES256CBCDecrypt::~AES256CBCDecrypt() { memset(iv, 0, sizeof(iv)); } - -AES128CBCEncrypt::AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : enc(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCEncrypt::~AES128CBCEncrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCEncrypt::Encrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCEncrypt(enc, iv, data, size, pad, out); -} - -AES128CBCDecrypt::AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : dec(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCDecrypt::~AES128CBCDecrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCDecrypt::Decrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCDecrypt(dec, iv, data, size, pad, out); -} diff --git a/src/crypto/aes.h b/src/crypto/aes.h index fdad70c593..e06c8de272 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -12,33 +12,8 @@ extern "C" { } static const int AES_BLOCKSIZE = 16; -static const int AES128_KEYSIZE = 16; static const int AES256_KEYSIZE = 32; -/** An encryption class for AES-128. */ -class AES128Encrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Encrypt(const unsigned char key[16]); - ~AES128Encrypt(); - void Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const; -}; - -/** A decryption class for AES-128. */ -class AES128Decrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Decrypt(const unsigned char key[16]); - ~AES128Decrypt(); - void Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const; -}; - /** An encryption class for AES-256. */ class AES256Encrypt { @@ -89,30 +64,4 @@ private: unsigned char iv[AES_BLOCKSIZE]; }; -class AES128CBCEncrypt -{ -public: - AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCEncrypt(); - int Encrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Encrypt enc; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - -class AES128CBCDecrypt -{ -public: - AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCDecrypt(); - int Decrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Decrypt dec; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - #endif // BITCOIN_CRYPTO_AES_H diff --git a/src/crypto/poly1305.cpp b/src/crypto/poly1305.cpp new file mode 100644 index 0000000000..8a86c9601c --- /dev/null +++ b/src/crypto/poly1305.cpp @@ -0,0 +1,141 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation by Andrew Moon +// poly1305-donna-unrolled.c from https://github.com/floodyberry/poly1305-donna + +#include <crypto/common.h> +#include <crypto/poly1305.h> + +#include <string.h> + +#define mul32x32_64(a,b) ((uint64_t)(a) * (b)) + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, const unsigned char key[POLY1305_KEYLEN]) { + uint32_t t0,t1,t2,t3; + uint32_t h0,h1,h2,h3,h4; + uint32_t r0,r1,r2,r3,r4; + uint32_t s1,s2,s3,s4; + uint32_t b, nb; + size_t j; + uint64_t t[5]; + uint64_t f0,f1,f2,f3; + uint64_t g0,g1,g2,g3,g4; + uint64_t c; + unsigned char mp[16]; + + /* clamp key */ + t0 = ReadLE32(key+0); + t1 = ReadLE32(key+4); + t2 = ReadLE32(key+8); + t3 = ReadLE32(key+12); + + /* precompute multipliers */ + r0 = t0 & 0x3ffffff; t0 >>= 26; t0 |= t1 << 6; + r1 = t0 & 0x3ffff03; t1 >>= 20; t1 |= t2 << 12; + r2 = t1 & 0x3ffc0ff; t2 >>= 14; t2 |= t3 << 18; + r3 = t2 & 0x3f03fff; t3 >>= 8; + r4 = t3 & 0x00fffff; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + /* init state */ + h0 = 0; + h1 = 0; + h2 = 0; + h3 = 0; + h4 = 0; + + /* full blocks */ + if (inlen < 16) goto poly1305_donna_atmost15bytes; +poly1305_donna_16bytes: + m += 16; + inlen -= 16; + + t0 = ReadLE32(m-16); + t1 = ReadLE32(m-12); + t2 = ReadLE32(m-8); + t3 = ReadLE32(m-4); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8) | (1 << 24); + + +poly1305_donna_mul: + t[0] = mul32x32_64(h0,r0) + mul32x32_64(h1,s4) + mul32x32_64(h2,s3) + mul32x32_64(h3,s2) + mul32x32_64(h4,s1); + t[1] = mul32x32_64(h0,r1) + mul32x32_64(h1,r0) + mul32x32_64(h2,s4) + mul32x32_64(h3,s3) + mul32x32_64(h4,s2); + t[2] = mul32x32_64(h0,r2) + mul32x32_64(h1,r1) + mul32x32_64(h2,r0) + mul32x32_64(h3,s4) + mul32x32_64(h4,s3); + t[3] = mul32x32_64(h0,r3) + mul32x32_64(h1,r2) + mul32x32_64(h2,r1) + mul32x32_64(h3,r0) + mul32x32_64(h4,s4); + t[4] = mul32x32_64(h0,r4) + mul32x32_64(h1,r3) + mul32x32_64(h2,r2) + mul32x32_64(h3,r1) + mul32x32_64(h4,r0); + + h0 = (uint32_t)t[0] & 0x3ffffff; c = (t[0] >> 26); + t[1] += c; h1 = (uint32_t)t[1] & 0x3ffffff; b = (uint32_t)(t[1] >> 26); + t[2] += b; h2 = (uint32_t)t[2] & 0x3ffffff; b = (uint32_t)(t[2] >> 26); + t[3] += b; h3 = (uint32_t)t[3] & 0x3ffffff; b = (uint32_t)(t[3] >> 26); + t[4] += b; h4 = (uint32_t)t[4] & 0x3ffffff; b = (uint32_t)(t[4] >> 26); + h0 += b * 5; + + if (inlen >= 16) goto poly1305_donna_16bytes; + + /* final bytes */ +poly1305_donna_atmost15bytes: + if (!inlen) goto poly1305_donna_finish; + + for (j = 0; j < inlen; j++) mp[j] = m[j]; + mp[j++] = 1; + for (; j < 16; j++) mp[j] = 0; + inlen = 0; + + t0 = ReadLE32(mp+0); + t1 = ReadLE32(mp+4); + t2 = ReadLE32(mp+8); + t3 = ReadLE32(mp+12); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8); + + goto poly1305_donna_mul; + +poly1305_donna_finish: + b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; b = h1 >> 26; h1 = h1 & 0x3ffffff; + h2 += b; b = h2 >> 26; h2 = h2 & 0x3ffffff; + h3 += b; b = h3 >> 26; h3 = h3 & 0x3ffffff; + h4 += b; b = h4 >> 26; h4 = h4 & 0x3ffffff; + h0 += b * 5; b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; + + g0 = h0 + 5; b = g0 >> 26; g0 &= 0x3ffffff; + g1 = h1 + b; b = g1 >> 26; g1 &= 0x3ffffff; + g2 = h2 + b; b = g2 >> 26; g2 &= 0x3ffffff; + g3 = h3 + b; b = g3 >> 26; g3 &= 0x3ffffff; + g4 = h4 + b - (1 << 26); + + b = (g4 >> 31) - 1; + nb = ~b; + h0 = (h0 & nb) | (g0 & b); + h1 = (h1 & nb) | (g1 & b); + h2 = (h2 & nb) | (g2 & b); + h3 = (h3 & nb) | (g3 & b); + h4 = (h4 & nb) | (g4 & b); + + f0 = ((h0 ) | (h1 << 26)) + (uint64_t)ReadLE32(&key[16]); + f1 = ((h1 >> 6) | (h2 << 20)) + (uint64_t)ReadLE32(&key[20]); + f2 = ((h2 >> 12) | (h3 << 14)) + (uint64_t)ReadLE32(&key[24]); + f3 = ((h3 >> 18) | (h4 << 8)) + (uint64_t)ReadLE32(&key[28]); + + WriteLE32(&out[ 0], f0); f1 += (f0 >> 32); + WriteLE32(&out[ 4], f1); f2 += (f1 >> 32); + WriteLE32(&out[ 8], f2); f3 += (f2 >> 32); + WriteLE32(&out[12], f3); +} diff --git a/src/crypto/poly1305.h b/src/crypto/poly1305.h new file mode 100644 index 0000000000..1598b013b9 --- /dev/null +++ b/src/crypto/poly1305.h @@ -0,0 +1,17 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_POLY1305_H +#define BITCOIN_CRYPTO_POLY1305_H + +#include <stdint.h> +#include <stdlib.h> + +#define POLY1305_KEYLEN 32 +#define POLY1305_TAGLEN 16 + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, + const unsigned char key[POLY1305_KEYLEN]); + +#endif // BITCOIN_CRYPTO_POLY1305_H diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 8a76021a5b..eeec6dec25 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -23,11 +23,33 @@ public: void DummyWalletInit::AddWalletOptions() const { - std::vector<std::string> opts = {"-addresstype", "-changetype", "-disablewallet", "-discardfee=<amt>", "-fallbackfee=<amt>", - "-keypool=<n>", "-mintxfee=<amt>", "-paytxfee=<amt>", "-rescan", "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=<n>", - "-upgradewallet", "-wallet=<path>", "-walletbroadcast", "-walletdir=<dir>", "-walletnotify=<cmd>", "-walletrbf", "-zapwallettxes=<mode>", - "-dblogsize=<n>", "-flushwallet", "-privdb", "-walletrejectlongchains"}; - gArgs.AddHiddenArgs(opts); + gArgs.AddHiddenArgs({ + "-addresstype", + "-avoidpartialspends", + "-changetype", + "-disablewallet", + "-discardfee=<amt>", + "-fallbackfee=<amt>", + "-keypool=<n>", + "-maxtxfee=<amt>", + "-mintxfee=<amt>", + "-paytxfee=<amt>", + "-rescan", + "-salvagewallet", + "-spendzeroconfchange", + "-txconfirmtarget=<n>", + "-upgradewallet", + "-wallet=<path>", + "-walletbroadcast", + "-walletdir=<dir>", + "-walletnotify=<cmd>", + "-walletrbf", + "-zapwallettxes=<mode>", + "-dblogsize=<n>", + "-flushwallet", + "-privdb", + "-walletrejectlongchains", + }); } const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 index 0000000000..8a8f7b681c --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdexcept> + +#include <flatfile.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : + m_dir(std::move(dir)), + m_prefix(prefix), + m_chunk_size(chunk_size) +{ + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +std::string FlatFilePos::ToString() const +{ + return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos); +} + +fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const +{ + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} + +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) +{ + if (pos.IsNull()) { + return nullptr; + } + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+"); + if (!file && !read_only) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + return file; +} + +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) +{ + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} + +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) +{ + FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 index 0000000000..374ceff411 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,96 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include <string> + +#include <fs.h> +#include <serialize.h> + +struct FlatFilePos +{ + int nFile; + unsigned int nPos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nPos)); + } + + FlatFilePos() : nFile(-1), nPos(0) {} + + FlatFilePos(int nFileIn, unsigned int nPosIn) : + nFile(nFileIn), + nPos(nPosIn) + {} + + friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { + return (a.nFile == b.nFile && a.nPos == b.nPos); + } + + friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { + return !(a == b); + } + + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } + + std::string ToString() const; +}; + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates + * access to and efficient management of these files. + */ +class FlatFileSeq +{ +private: + const fs::path m_dir; + const char* const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this amount. + */ + FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const FlatFilePos& pos) const; + + /** Open a handle to the file at the given position. */ + FILE* Open(const FlatFilePos& pos, bool read_only = false); + + /** + * Allocate additional space in a file after the given starting position. The amount allocated + * will be the minimum multiple of the sequence chunk size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const FlatFilePos& pos, bool finalize = false); +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/fs.cpp b/src/fs.cpp index f937f64e04..7b422b8d70 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -3,7 +3,9 @@ #ifndef WIN32 #include <fcntl.h> #else +#ifndef NOMINMAX #define NOMINMAX +#endif #include <codecvt> #include <windows.h> #endif diff --git a/src/httpserver.cpp b/src/httpserver.cpp index b9ca037c9d..63639fa3e0 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -6,6 +6,7 @@ #include <chainparamsbase.h> #include <compat.h> +#include <util/threadnames.h> #include <util/system.h> #include <util/strencodings.h> #include <netbase.h> @@ -17,7 +18,7 @@ #include <memory> #include <stdio.h> #include <stdlib.h> -#include <string.h> +#include <string> #include <sys/types.h> #include <sys/stat.h> @@ -284,7 +285,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) /** Event dispatcher thread */ static bool ThreadHTTP(struct event_base* base) { - RenameThread("bitcoin-http"); + util::ThreadRename("http"); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -335,9 +336,9 @@ static bool HTTPBindAddresses(struct evhttp* http) } /** Simple wrapper to set thread name and run work queue */ -static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue) +static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { - RenameThread("bitcoin-httpworker"); + util::ThreadRename(strprintf("httpworker.%i", worker_num)); queue->Run(); } @@ -430,7 +431,7 @@ void StartHTTPServer() threadHTTP = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { - g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue); + g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); } } @@ -655,15 +656,3 @@ void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) pathHandlers.erase(i); } } - -std::string urlDecode(const std::string &urlEncoded) { - std::string res; - if (!urlEncoded.empty()) { - char *decoded = evhttp_uridecode(urlEncoded.c_str(), false, nullptr); - if (decoded) { - res = std::string(decoded); - free(decoded); - } - } - return res; -} diff --git a/src/httpserver.h b/src/httpserver.h index 63f96734f8..7943f0094b 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -148,6 +148,4 @@ private: struct event* ev; }; -std::string urlDecode(const std::string &urlEncoded); - #endif // BITCOIN_HTTPSERVER_H diff --git a/src/index/base.cpp b/src/index/base.cpp index f6f59572ce..9e48f0bd27 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -41,9 +41,9 @@ bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const return success; } -bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator) +void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) { - return Write(DB_BEST_BLOCK, locator); + batch.Write(DB_BEST_BLOCK, locator); } BaseIndex::~BaseIndex() @@ -95,7 +95,11 @@ void BaseIndex::ThreadSync() int64_t last_locator_write_time = 0; while (true) { if (m_interrupt) { - WriteBestBlock(pindex); + m_best_block_index = pindex; + // No need to handle errors in Commit. If it fails, the error will be already be + // logged. The best way to recover is to continue, as index cannot be corrupted by + // a missed commit to disk for an advanced index state. + Commit(); return; } @@ -103,11 +107,17 @@ void BaseIndex::ThreadSync() LOCK(cs_main); const CBlockIndex* pindex_next = NextSyncBlock(pindex); if (!pindex_next) { - WriteBestBlock(pindex); m_best_block_index = pindex; m_synced = true; + // No need to handle errors in Commit. See rationale above. + Commit(); break; } + if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { + FatalError("%s: Failed to rewind index %s to a previous chain tip", + __func__, GetName()); + return; + } pindex = pindex_next; } @@ -119,8 +129,10 @@ void BaseIndex::ThreadSync() } if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - WriteBestBlock(pindex); + m_best_block_index = pindex; last_locator_write_time = current_time; + // No need to handle errors in Commit. See rationale above. + Commit(); } CBlock block; @@ -144,12 +156,35 @@ void BaseIndex::ThreadSync() } } -bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) +bool BaseIndex::Commit() +{ + CDBBatch batch(GetDB()); + if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) { + return error("%s: Failed to commit latest %s state", __func__, GetName()); + } + return true; +} + +bool BaseIndex::CommitInternal(CDBBatch& batch) { LOCK(cs_main); - if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { - return error("%s: Failed to write locator to disk", __func__); + GetDB().WriteBestBlock(batch, chainActive.GetLocator(m_best_block_index)); + return true; +} + +bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip == m_best_block_index); + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + // In the case of a reorg, ensure persisted block locator is not stale. + m_best_block_index = new_tip; + if (!Commit()) { + // If commit fails, revert the best block index to avoid corruption. + m_best_block_index = current_tip; + return false; } + return true; } @@ -180,6 +215,11 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const best_block_index->GetBlockHash().ToString()); return; } + if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) { + FatalError("%s: Failed to rewind index %s to a previous chain tip", + __func__, GetName()); + return; + } } if (WriteBlock(*block, pindex)) { @@ -224,9 +264,10 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) return; } - if (!GetDB().WriteBestBlock(locator)) { - error("%s: Failed to write locator to disk", __func__); - } + // No need to handle errors in Commit. If it fails, the error will be already be logged. The + // best way to recover is to continue, as index cannot be corrupted by a missed commit to disk + // for an advanced index state. + Commit(); } bool BaseIndex::BlockUntilSyncedToCurrentChain() diff --git a/src/index/base.h b/src/index/base.h index 04ee6e6cc2..31acbed0c1 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -32,7 +32,7 @@ protected: bool ReadBestBlock(CBlockLocator& locator) const; /// Write block locator of the chain that the txindex is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); + void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator); }; private: @@ -54,8 +54,15 @@ private: /// over and the sync thread exits. void ThreadSync(); - /// Write the current chain block locator to the DB. - bool WriteBestBlock(const CBlockIndex* block_index); + /// Write the current index state (eg. chain block locator and subclass-specific items) to disk. + /// + /// Recommendations for error handling: + /// If called on a successor of the previous committed best block in the index, the index can + /// continue processing without risk of corruption, though the index state will need to catch up + /// from further behind on reboot. If the new state is not a successor of the previous state (due + /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up + /// getting corrupted. + bool Commit(); protected: void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, @@ -69,6 +76,14 @@ protected: /// Write update index entries for a newly connected block. virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + /// Virtual method called internally by Commit that can be overridden to atomically + /// commit more index state. + virtual bool CommitInternal(CDBBatch& batch); + + /// Rewind index to an earlier chain tip during a chain reorg. The tip must + /// be an ancestor of the current best block. + virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + virtual DB& GetDB() const = 0; /// Get the name of the index for display in logs. diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp new file mode 100644 index 0000000000..20f33baf2c --- /dev/null +++ b/src/index/blockfilterindex.cpp @@ -0,0 +1,467 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <map> + +#include <dbwrapper.h> +#include <index/blockfilterindex.h> +#include <util/system.h> +#include <validation.h> + +/* The index database stores three items for each block: the disk location of the encoded filter, + * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by + * height, and those belonging to blocks that have been reorganized out of the active chain are + * indexed by block hash. This ensures that filter data for any block that becomes part of the + * active chain can always be retrieved, alleviating timing concerns. + * + * The filters themselves are stored in flat files and referenced by the LevelDB entries. This + * minimizes the amount of data written to LevelDB and keeps the database values constant size. The + * disk location of the next block filter to be written (represented as a FlatFilePos) is stored + * under the DB_FILTER_POS key. + * + * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented + * as big-endian so that sequential reads of filters by height are fast. + * Keys for the hash index have the type [DB_BLOCK_HASH, uint256]. + */ +constexpr char DB_BLOCK_HASH = 's'; +constexpr char DB_BLOCK_HEIGHT = 't'; +constexpr char DB_FILTER_POS = 'P'; + +constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for fltr?????.dat files */ +constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB + +namespace { + +struct DBVal { + uint256 hash; + uint256 header; + FlatFilePos pos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(hash); + READWRITE(header); + READWRITE(pos); + } +}; + +struct DBHeightKey { + int height; + + DBHeightKey() : height(0) {} + DBHeightKey(int height_in) : height(height_in) {} + + template<typename Stream> + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template<typename Stream> + void Unserialize(Stream& s) + { + char prefix = ser_readdata8(s); + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for block filter index DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 hash; + + DBHashKey(const uint256& hash_in) : hash(hash_in) {} + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + char prefix = DB_BLOCK_HASH; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for block filter index DB hash key"); + } + + READWRITE(hash); + } +}; + +}; // namespace + +static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes; + +BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory, bool f_wipe) + : m_filter_type(filter_type) +{ + const std::string& filter_name = BlockFilterTypeName(filter_type); + if (filter_name.empty()) throw std::invalid_argument("unknown filter_type"); + + fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name; + fs::create_directories(path); + + m_name = filter_name + " block filter index"; + m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); + m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE); +} + +bool BlockFilterIndex::Init() +{ + if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { + // Check that the cause of the read failure is that the key does not exist. Any other errors + // indicate database corruption or a disk failure, and starting the index would cause + // further corruption. + if (m_db->Exists(DB_FILTER_POS)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + + // If the DB_FILTER_POS is not set, then initialize to the first location. + m_next_filter_pos.nFile = 0; + m_next_filter_pos.nPos = 0; + } + return BaseIndex::Init(); +} + +bool BlockFilterIndex::CommitInternal(CDBBatch& batch) +{ + const FlatFilePos& pos = m_next_filter_pos; + + // Flush current filter file to disk. + CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + return error("%s: Failed to open filter file %d", __func__, pos.nFile); + } + if (!FileCommit(file.Get())) { + return error("%s: Failed to commit filter file %d", __func__, pos.nFile); + } + + batch.Write(DB_FILTER_POS, pos); + return BaseIndex::CommitInternal(batch); +} + +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +{ + CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return false; + } + + uint256 block_hash; + std::vector<unsigned char> encoded_filter; + try { + filein >> block_hash >> encoded_filter; + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + } + catch (const std::exception& e) { + return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); + } + + return true; +} + +size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter) +{ + assert(filter.GetFilterType() == GetFilterType()); + + size_t data_size = + GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) + + GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION); + + // If writing the filter would overflow the file, flush and move to the next one. + if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { + CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (last_file.IsNull()) { + LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); + return 0; + } + if (!TruncateFile(last_file.Get(), pos.nPos)) { + LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile); + return 0; + } + if (!FileCommit(last_file.Get())) { + LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile); + return 0; + } + + pos.nFile++; + pos.nPos = 0; + } + + // Pre-allocate sufficient space for filter data. + bool out_of_space; + m_filter_fileseq->Allocate(pos, data_size, out_of_space); + if (out_of_space) { + LogPrintf("%s: out of disk space\n", __func__); + return 0; + } + + CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); + return 0; + } + + fileout << filter.GetBlockHash() << filter.GetEncodedFilter(); + return data_size; +} + +bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + uint256 prev_header; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair<uint256, DBVal> read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + + prev_header = read_out.second.header; + } + + BlockFilter filter(m_filter_type, block, block_undo); + + size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); + if (bytes_written == 0) return false; + + std::pair<uint256, DBVal> value; + value.first = pindex->GetBlockHash(); + value.second.hash = filter.GetHash(); + value.second.header = filter.ComputeHeader(prev_header); + value.second.pos = m_next_filter_pos; + + if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + return false; + } + + m_next_filter_pos.nPos += bytes_written; + return true; +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key(start_height); + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair<uint256, DBVal> value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); + + // During a reorg, we need to copy all filters for blocks that are getting disconnected from the + // height index to the hash index so we can still find them when the height index entries are + // overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind. + // But since this creates new references to the filter, the position should get updated here + // atomically as well in case Commit fails. + batch.Write(DB_FILTER_POS, m_next_filter_pos); + if (!m_db->WriteBatch(batch)) return false; + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value there matches the + // block hash. This should be the case if the block is on the active chain. + std::pair<uint256, DBVal> read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the result will be stored in + // the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height, + const CBlockIndex* stop_index, std::vector<DBVal>& results) +{ + if (start_height < 0) { + return error("%s: start height (%d) is negative", __func__, start_height); + } + if (start_height > stop_index->nHeight) { + return error("%s: start height (%d) is greater than stop height (%d)", + __func__, start_height, stop_index->nHeight); + } + + size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1); + std::vector<std::pair<uint256, DBVal>> values(results_size); + + DBHeightKey key(start_height); + std::unique_ptr<CDBIterator> db_it(db.NewIterator()); + db_it->Seek(DBHeightKey(start_height)); + for (int height = start_height; height <= stop_index->nHeight; ++height) { + if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) { + return false; + } + + size_t i = static_cast<size_t>(height - start_height); + if (!db_it->GetValue(values[i])) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + db_it->Next(); + } + + results.resize(results_size); + + // Iterate backwards through block indexes collecting results in order to access the block hash + // of each entry in case we need to look it up in the hash index. + for (const CBlockIndex* block_index = stop_index; + block_index && block_index->nHeight >= start_height; + block_index = block_index->pprev) { + uint256 block_hash = block_index->GetBlockHash(); + + size_t i = static_cast<size_t>(block_index->nHeight - start_height); + if (block_hash == values[i].first) { + results[i] = std::move(values[i].second); + continue; + } + + if (!db.Read(DBHashKey(block_hash), results[i])) { + return error("%s: unable to read value in %s at key (%c, %s)", + __func__, index_name, DB_BLOCK_HASH, block_hash.ToString()); + } + } + + return true; +} + +bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + return ReadFilterFromDisk(entry.pos, filter_out); +} + +bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + header_out = entry.header; + return true; +} + +bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index, + std::vector<BlockFilter>& filters_out) const +{ + std::vector<DBVal> entries; + if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) { + return false; + } + + filters_out.resize(entries.size()); + auto filter_pos_it = filters_out.begin(); + for (const auto& entry : entries) { + if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + return false; + } + ++filter_pos_it; + } + + return true; +} + +bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index, + std::vector<uint256>& hashes_out) const + +{ + std::vector<DBVal> entries; + if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) { + return false; + } + + hashes_out.clear(); + hashes_out.reserve(entries.size()); + for (const auto& entry : entries) { + hashes_out.push_back(entry.hash); + } + return true; +} + +BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type) +{ + auto it = g_filter_indexes.find(filter_type); + return it != g_filter_indexes.end() ? &it->second : nullptr; +} + +void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn) +{ + for (auto& entry : g_filter_indexes) fn(entry.second); +} + +bool InitBlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory, bool f_wipe) +{ + auto result = g_filter_indexes.emplace(std::piecewise_construct, + std::forward_as_tuple(filter_type), + std::forward_as_tuple(filter_type, + n_cache_size, f_memory, f_wipe)); + return result.second; +} + +bool DestroyBlockFilterIndex(BlockFilterType filter_type) +{ + return g_filter_indexes.erase(filter_type); +} + +void DestroyAllBlockFilterIndexes() +{ + g_filter_indexes.clear(); +} diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h new file mode 100644 index 0000000000..436d52515f --- /dev/null +++ b/src/index/blockfilterindex.h @@ -0,0 +1,94 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_BLOCKFILTERINDEX_H +#define BITCOIN_INDEX_BLOCKFILTERINDEX_H + +#include <blockfilter.h> +#include <chain.h> +#include <flatfile.h> +#include <index/base.h> + +/** + * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of + * blocks by height. An index is constructed for each supported filter type with its own database + * (ie. filter data for different types are stored in separate databases). + * + * This index is used to serve BIP 157 net requests. + */ +class BlockFilterIndex final : public BaseIndex +{ +private: + BlockFilterType m_filter_type; + std::string m_name; + std::unique_ptr<BaseIndex::DB> m_db; + + FlatFilePos m_next_filter_pos; + std::unique_ptr<FlatFileSeq> m_filter_fileseq; + + bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); + +protected: + bool Init() override; + + bool CommitInternal(CDBBatch& batch) override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return m_name.c_str(); } + +public: + /** Constructs the index, which becomes available to be queried. */ + explicit BlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + BlockFilterType GetFilterType() const { return m_filter_type; } + + /** Get a single filter by block. */ + bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const; + + /** Get a single filter header by block. */ + bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const; + + /** Get a range of filters between two heights on a chain. */ + bool LookupFilterRange(int start_height, const CBlockIndex* stop_index, + std::vector<BlockFilter>& filters_out) const; + + /** Get a range of filter hashes between two heights on a chain. */ + bool LookupFilterHashRange(int start_height, const CBlockIndex* stop_index, + std::vector<uint256>& hashes_out) const; +}; + +/** + * Get a block filter index by type. Returns nullptr if index has not been initialized or was + * already destroyed. + */ +BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type); + +/** Iterate over all running block filter indexes, invoking fn on each. */ +void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn); + +/** + * Initialize a block filter index for the given type if one does not already exist. Returns true if + * a new index is created and false if one has already been initialized. + */ +bool InitBlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + +/** + * Destroy the block filter index with the given type. Returns false if no such index exists. This + * just releases the allocated memory and closes the database connection, it does not delete the + * index data. + */ +bool DestroyBlockFilterIndex(BlockFilterType filter_type); + +/** Destroy all open block filter indexes. */ +void DestroyAllBlockFilterIndexes(); + +#endif // BITCOIN_INDEX_BLOCKFILTERINDEX_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 10bc8419dd..7367ec7cb6 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public CDiskBlockPos +struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header @@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); + READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } CDiskTxPos() { @@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos } void SetNull() { - CDiskBlockPos::SetNull(); + FlatFilePos::SetNull(); nTxOffset = 0; } }; diff --git a/src/init.cpp b/src/init.cpp index 8876b54fde..92b3c9510a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -12,14 +12,15 @@ #include <addrman.h> #include <amount.h> #include <banman.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <compat/sanity.h> #include <consensus/validation.h> #include <fs.h> #include <httpserver.h> #include <httprpc.h> +#include <index/blockfilterindex.h> #include <interfaces/chain.h> #include <index/txindex.h> #include <key.h> @@ -31,6 +32,7 @@ #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> +#include <policy/settings.h> #include <rpc/server.h> #include <rpc/register.h> #include <rpc/blockchain.h> @@ -39,6 +41,7 @@ #include <script/sigcache.h> #include <scheduler.h> #include <shutdown.h> +#include <util/threadnames.h> #include <timedata.h> #include <txdb.h> #include <txmempool.h> @@ -46,6 +49,7 @@ #include <ui_interface.h> #include <util/system.h> #include <util/moneystr.h> +#include <util/validation.h> #include <validationinterface.h> #include <warnings.h> #include <walletinitinterface.h> @@ -63,7 +67,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/thread.hpp> -#include <openssl/crypto.h> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -189,6 +192,7 @@ void Interrupt() if (g_txindex) { g_txindex->Interrupt(); } + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); } void Shutdown(InitInterfaces& interfaces) @@ -203,7 +207,7 @@ void Shutdown(InitInterfaces& interfaces) /// for example if the data directory was found to be locked. /// Be sure that anything that writes files or flushes caches only does this if the respective /// module was initialized. - RenameThread("bitcoin-shutoff"); + util::ThreadRename("shutoff"); mempool.AddTransactionsUpdated(1); StopHTTPRPC(); @@ -220,6 +224,7 @@ void Shutdown(InitInterfaces& interfaces) if (peerLogic) UnregisterValidationInterface(peerLogic.get()); if (g_connman) g_connman->Stop(); if (g_txindex) g_txindex->Stop(); + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); StopTorControl(); @@ -234,9 +239,10 @@ void Shutdown(InitInterfaces& interfaces) g_connman.reset(); g_banman.reset(); g_txindex.reset(); + DestroyAllBlockFilterIndexes(); - if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(); + if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + DumpMempool(::mempool); } if (fFeeEstimatesInitialized) @@ -339,14 +345,15 @@ static void registerSignalHandler(int signal, void(*handler)(int)) } #endif +static boost::signals2::connection rpc_notify_block_change_connection; static void OnRPCStarted() { - uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); } static void OnRPCStopped() { - uiInterface.NotifyBlockTip_disconnect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection.disconnect(); RPCNotifyBlockChange(false, nullptr); g_best_block_cv.notify_all(); LogPrint(BCLog::RPC, "RPC stopped.\n"); @@ -404,6 +411,10 @@ void SetupServerArgs() hidden_args.emplace_back("-sysperms"); #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS); + gArgs.AddArg("-blockfilterindex=<type>", + strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) + + " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.", + false, OptionsCategory::OPTIONS); gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION); gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION); @@ -496,12 +507,11 @@ void SetupServerArgs() gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), false, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)", true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), true, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction or raw transaction; setting this too low may abort large transactions (default: %s)", - CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", false, OptionsCategory::DEBUG_TEST); @@ -511,7 +521,7 @@ void SetupServerArgs() gArgs.AddArg("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !testnetChainParams->RequireStandard()), true, OptionsCategory::NODE_RELAY); gArgs.AddArg("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE)), true, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), true, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), true, OptionsCategory::NODE_RELAY); gArgs.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), false, OptionsCategory::NODE_RELAY); gArgs.AddArg("-datacarrier", strprintf("Relay and mine data carrier transactions (default: %u)", DEFAULT_ACCEPT_DATACARRIER), false, OptionsCategory::NODE_RELAY); gArgs.AddArg("-datacarriersize", strprintf("Maximum size of data in data carrier transactions we relay and mine (default: %u)", MAX_OP_RETURN_RELAY), false, OptionsCategory::NODE_RELAY); @@ -658,7 +668,7 @@ static void CleanupBlockRevFiles() static void ThreadImport(std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); - RenameThread("bitcoin-loadblk"); + util::ThreadRename("loadblk"); ScheduleBatchPriority(); { @@ -668,8 +678,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (fReindex) { int nFile = 0; while (true) { - CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) + FlatFilePos pos(nFile, 0); + if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -725,9 +735,9 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } } // End scope of CImportingNow if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - LoadMempool(); + LoadMempool(::mempool); } - g_is_mempool_loaded = !ShutdownRequested(); + ::mempool.SetIsLoaded(!ShutdownRequested()); } /** Sanity checks @@ -828,19 +838,6 @@ void InitParameterInteraction() if (gArgs.SoftSetBoolArg("-whitelistrelay", true)) LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> setting -whitelistrelay=1\n", __func__); } - - // Warn if network-specific options (-addnode, -connect, etc) are - // specified in default section of config file, but not overridden - // on the command line or in this network's section of the config file. - std::string network = gArgs.GetChainName(); - for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { - InitWarning(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); - } - - // Warn if unrecognized section name are present in the config file. - for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf(_("Section [%s] is not recognized."), section)); - } } static std::string ResolveErrMsg(const char * const optname, const std::string& strBind) @@ -867,6 +864,7 @@ void InitLogging() LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); + LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); @@ -886,6 +884,7 @@ int nUserMaxConnections; int nFD; ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); int64_t peer_connect_timeout; +std::vector<BlockFilterType> g_enabled_filter_types; } // namespace @@ -950,14 +949,46 @@ bool AppInitParameterInteraction() // also see: InitParameterInteraction() + // Warn if network-specific options (-addnode, -connect, etc) are + // specified in default section of config file, but not overridden + // on the command line or in this network's section of the config file. + std::string network = gArgs.GetChainName(); + for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { + return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); + } + + // Warn if unrecognized section name are present in the config file. + for (const auto& section : gArgs.GetUnrecognizedSections()) { + InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); + } + if (!fs::is_directory(GetBlocksDir())) { return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str())); } + // parse and validate enabled filter types + std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); + if (blockfilterindex_value == "" || blockfilterindex_value == "1") { + g_enabled_filter_types = AllBlockFilterTypes(); + } else if (blockfilterindex_value != "0") { + const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex"); + g_enabled_filter_types.reserve(names.size()); + for (const auto& name : names) { + BlockFilterType filter_type; + if (!BlockFilterTypeByName(name, filter_type)) { + return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name)); + } + g_enabled_filter_types.push_back(filter_type); + } + } + // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); + if (!g_enabled_filter_types.empty()) { + return InitError(_("Prune mode is incompatible with -blockfilterindex.")); + } } // -bind and -whitebind can't be set when not listening @@ -1121,22 +1152,6 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } - // This is required by both the wallet and node - if (gArgs.IsArgSet("-maxtxfee")) - { - CAmount nMaxFee = 0; - if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) - return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); - if (nMaxFee > HIGH_MAX_TX_FEE) - InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); - maxTxFee = nMaxFee; - if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) - { - return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), - gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); - } - } - fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); @@ -1274,7 +1289,7 @@ bool AppInitMain(InitInterfaces& interfaces) LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); if (nScriptCheckThreads) { for (int i=0; i<nScriptCheckThreads-1; i++) - threadGroup.create_thread(&ThreadScriptCheck); + threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); } // Start the lightweight task scheduler thread @@ -1448,6 +1463,13 @@ bool AppInitMain(InitInterfaces& interfaces) nTotalCache -= nBlockTreeDBCache; int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); nTotalCache -= nTxIndexCache; + int64_t filter_index_cache = 0; + if (!g_enabled_filter_types.empty()) { + size_t n_indexes = g_enabled_filter_types.size(); + int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20); + filter_index_cache = max_cache / n_indexes; + nTotalCache -= filter_index_cache * n_indexes; + } int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1458,6 +1480,10 @@ bool AppInitMain(InitInterfaces& interfaces) if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); } + for (BlockFilterType filter_type : g_enabled_filter_types) { + LogPrintf("* Using %.1f MiB for %s block filter index database\n", + filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); + } LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); @@ -1468,11 +1494,11 @@ bool AppInitMain(InitInterfaces& interfaces) uiInterface.InitMessage(_("Loading block index...")); - LOCK(cs_main); - do { const int64_t load_block_index_start_time = GetTimeMillis(); + bool is_coinsview_empty; try { + LOCK(cs_main); UnloadBlockIndex(); pcoinsTip.reset(); pcoinsdbview.reset(); @@ -1544,7 +1570,7 @@ bool AppInitMain(InitInterfaces& interfaces) // The on-disk coinsdb is now in a good state, create the cache pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); - bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); + is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); if (!is_coinsview_empty) { // LoadChainTip sets chainActive based on pcoinsTip's best block if (!LoadChainTip(chainparams)) { @@ -1553,18 +1579,25 @@ bool AppInitMain(InitInterfaces& interfaces) } assert(chainActive.Tip() != nullptr); } + } catch (const std::exception& e) { + LogPrintf("%s\n", e.what()); + strLoadError = _("Error opening block database"); + break; + } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on chainActive, and drops block data in - // mapBlockIndex based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...")); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); - break; - } + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on chainActive, and drops block data in + // mapBlockIndex based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...")); + if (!RewindBlockIndex(chainparams)) { + strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); + break; } + } + try { + LOCK(cs_main); if (!is_coinsview_empty) { uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { @@ -1638,6 +1671,11 @@ bool AppInitMain(InitInterfaces& interfaces) g_txindex->Start(); } + for (const auto& filter_type : g_enabled_filter_types) { + InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex); + GetBlockFilterIndex(filter_type)->Start(); + } + // ********************************************************* Step 9: load wallet for (const auto& client : interfaces.chain_clients) { if (!client->load()) { @@ -1669,19 +1707,20 @@ bool AppInitMain(InitInterfaces& interfaces) // ********************************************************* Step 11: import blocks - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { + if (!CheckDiskSpace(GetDataDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) { + if (!CheckDiskSpace(GetBlocksDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. // No locking, as this happens before any background thread is started. + boost::signals2::connection block_notify_genesis_wait_connection; if (chainActive.Tip() == nullptr) { - uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); } else { fHaveGenesis = true; } @@ -1705,7 +1744,7 @@ bool AppInitMain(InitInterfaces& interfaces) while (!fHaveGenesis && !ShutdownRequested()) { g_genesis_wait_cv.wait_for(lock, std::chrono::milliseconds(500)); } - uiInterface.NotifyBlockTip_disconnect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection.disconnect(); } if (ShutdownRequested()) { diff --git a/src/interfaces/README.md b/src/interfaces/README.md index 57d41df746..f77d172153 100644 --- a/src/interfaces/README.md +++ b/src/interfaces/README.md @@ -2,9 +2,9 @@ The following interfaces are defined here: -* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437), [#14711](https://github.com/bitcoin/bitcoin/pull/14711), [#15288](https://github.com/bitcoin/bitcoin/pull/15288), and [#10973](https://github.com/bitcoin/bitcoin/pull/10973). -* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437). * [`Node`](node.h) — used by GUI to start & stop bitcoin node. Added in [#10244](https://github.com/bitcoin/bitcoin/pull/10244). diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index da810bc5e6..617be3ca71 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -6,11 +6,30 @@ #include <chain.h> #include <chainparams.h> +#include <interfaces/handler.h> +#include <interfaces/wallet.h> +#include <net.h> +#include <node/coin.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <policy/settings.h> #include <primitives/block.h> +#include <primitives/transaction.h> +#include <protocol.h> +#include <rpc/protocol.h> +#include <rpc/server.h> +#include <shutdown.h> #include <sync.h> +#include <threadsafety.h> +#include <timedata.h> +#include <txmempool.h> +#include <ui_interface.h> #include <uint256.h> +#include <univalue.h> #include <util/system.h> #include <validation.h> +#include <validationinterface.h> #include <memory> #include <utility> @@ -65,29 +84,15 @@ class LockImpl : public Chain::Lock CBlockIndex* block = ::chainActive[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } - Optional<int> findFirstBlockWithTime(int64_t time, uint256* hash) override + Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override { - CBlockIndex* block = ::chainActive.FindEarliestAtLeast(time); + CBlockIndex* block = ::chainActive.FindEarliestAtLeast(time, height); if (block) { if (hash) *hash = block->GetBlockHash(); return block->nHeight; } return nullopt; } - Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height) override - { - // TODO: Could update CChain::FindEarliestAtLeast() to take a height - // parameter and use it with std::lower_bound() to make this - // implementation more efficient and allow combining - // findFirstBlockWithTime and findFirstBlockWithTimeAndHeight into one - // method. - for (CBlockIndex* block = ::chainActive[height]; block; block = ::chainActive.Next(block)) { - if (block->GetBlockTime() >= time) { - return block->nHeight; - } - } - return nullopt; - } Optional<int> findPruned(int start_height, Optional<int> stop_height) override { if (::fPruneMode) { @@ -117,12 +122,6 @@ class LockImpl : public Chain::Lock } return nullopt; } - bool isPotentialTip(const uint256& hash) override - { - if (::chainActive.Tip()->GetBlockHash() == hash) return true; - CBlockIndex* block = LookupBlockIndex(hash); - return block && block->GetAncestor(::chainActive.Height()) == ::chainActive.Tip(); - } CBlockLocator getTipLocator() override { return ::chainActive.GetLocator(); } Optional<int> findLocatorFork(const CBlockLocator& locator) override { @@ -132,6 +131,17 @@ class LockImpl : public Chain::Lock } return nullopt; } + bool checkFinalTx(const CTransaction& tx) override + { + LockAnnotation lock(::cs_main); + return CheckFinalTx(tx); + } + bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) override + { + LockAnnotation lock(::cs_main); + return AcceptToMemoryPool(::mempool, state, tx, nullptr /* missing inputs */, nullptr /* txn replaced */, + false /* bypass limits */, absurd_fee); + } }; class LockingStateImpl : public LockImpl, public UniqueLock<CCriticalSection> @@ -139,6 +149,88 @@ class LockingStateImpl : public LockImpl, public UniqueLock<CCriticalSection> using UniqueLock::UniqueLock; }; +class NotificationsHandlerImpl : public Handler, CValidationInterface +{ +public: + explicit NotificationsHandlerImpl(Chain& chain, Chain::Notifications& notifications) + : m_chain(chain), m_notifications(¬ifications) + { + RegisterValidationInterface(this); + } + ~NotificationsHandlerImpl() override { disconnect(); } + void disconnect() override + { + if (m_notifications) { + m_notifications = nullptr; + UnregisterValidationInterface(this); + } + } + void TransactionAddedToMempool(const CTransactionRef& tx) override + { + m_notifications->TransactionAddedToMempool(tx); + } + void TransactionRemovedFromMempool(const CTransactionRef& tx) override + { + m_notifications->TransactionRemovedFromMempool(tx); + } + void BlockConnected(const std::shared_ptr<const CBlock>& block, + const CBlockIndex* index, + const std::vector<CTransactionRef>& tx_conflicted) override + { + m_notifications->BlockConnected(*block, tx_conflicted); + } + void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + { + m_notifications->BlockDisconnected(*block); + } + void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override + { + m_notifications->UpdatedBlockTip(); + } + void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->ChainStateFlushed(locator); } + Chain& m_chain; + Chain::Notifications* m_notifications; +}; + +class RpcHandlerImpl : public Handler +{ +public: + RpcHandlerImpl(const CRPCCommand& command) : m_command(command), m_wrapped_command(&command) + { + m_command.actor = [this](const JSONRPCRequest& request, UniValue& result, bool last_handler) { + if (!m_wrapped_command) return false; + try { + return m_wrapped_command->actor(request, result, last_handler); + } catch (const UniValue& e) { + // If this is not the last handler and a wallet not found + // exception was thrown, return false so the next handler can + // try to handle the request. Otherwise, reraise the exception. + if (!last_handler) { + const UniValue& code = e["code"]; + if (code.isNum() && code.get_int() == RPC_WALLET_NOT_FOUND) { + return false; + } + } + throw; + } + }; + ::tableRPC.appendCommand(m_command.name, &m_command); + } + + void disconnect() override final + { + if (m_wrapped_command) { + m_wrapped_command = nullptr; + ::tableRPC.removeCommand(m_command.name, &m_command); + } + } + + ~RpcHandlerImpl() override { disconnect(); } + + CRPCCommand m_command; + const CRPCCommand* m_wrapped_command; +}; + class ChainImpl : public Chain { public: @@ -172,13 +264,107 @@ public: } return true; } + void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(coins); } double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + RBFTransactionState isRBFOptIn(const CTransaction& tx) override + { + LOCK(::mempool.cs); + return IsRBFOptIn(tx, ::mempool); + } + bool hasDescendantsInMempool(const uint256& txid) override + { + LOCK(::mempool.cs); + auto it = ::mempool.GetIter(txid); + return it && (*it)->GetCountWithDescendants() > 1; + } + void relayTransaction(const uint256& txid) override + { + CInv inv(MSG_TX, txid); + g_connman->ForEachNode([&inv](CNode* node) { node->PushInventory(inv); }); + } + void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override + { + ::mempool.GetTransactionAncestry(txid, ancestors, descendants); + } + bool checkChainLimits(const CTransactionRef& tx) override + { + LockPoints lp; + CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); + CTxMemPool::setEntries ancestors; + auto limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + auto limit_ancestor_size = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + std::string unused_error_string; + LOCK(::mempool.cs); + return ::mempool.CalculateMemPoolAncestors(entry, ancestors, limit_ancestor_count, limit_ancestor_size, + limit_descendant_count, limit_descendant_size, unused_error_string); + } + CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override + { + return ::feeEstimator.estimateSmartFee(num_blocks, calc, conservative); + } + unsigned int estimateMaxBlocks() override + { + return ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + } + CFeeRate mempoolMinFee() override + { + return ::mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + } + CFeeRate relayMinFee() override { return ::minRelayTxFee; } + CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } + CFeeRate relayDustFee() override { return ::dustRelayFee; } + bool getPruneMode() override { return ::fPruneMode; } + bool p2pEnabled() override { return g_connman != nullptr; } + bool isReadyToBroadcast() override { return !::fImporting && !::fReindex && !IsInitialBlockDownload(); } + bool isInitialBlockDownload() override { return IsInitialBlockDownload(); } + bool shutdownRequested() override { return ShutdownRequested(); } + int64_t getAdjustedTime() override { return GetAdjustedTime(); } + void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } + void initWarning(const std::string& message) override { InitWarning(message); } + void initError(const std::string& message) override { InitError(message); } + void loadWallet(std::unique_ptr<Wallet> wallet) override { ::uiInterface.LoadWallet(wallet); } + void showProgress(const std::string& title, int progress, bool resume_possible) override + { + ::uiInterface.ShowProgress(title, progress, resume_possible); + } + std::unique_ptr<Handler> handleNotifications(Notifications& notifications) override + { + return MakeUnique<NotificationsHandlerImpl>(*this, notifications); + } + void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) override + { + if (!old_tip.IsNull()) { + LOCK(::cs_main); + if (old_tip == ::chainActive.Tip()->GetBlockHash()) return; + CBlockIndex* block = LookupBlockIndex(old_tip); + if (block && block->GetAncestor(::chainActive.Height()) == ::chainActive.Tip()) return; + } + SyncWithValidationInterfaceQueue(); + } + std::unique_ptr<Handler> handleRpc(const CRPCCommand& command) override + { + return MakeUnique<RpcHandlerImpl>(command); + } + bool rpcEnableDeprecated(const std::string& method) override { return IsDeprecatedRPCEnabled(method); } + void rpcRunLater(const std::string& name, std::function<void()> fn, int64_t seconds) override + { + RPCRunLater(name, std::move(fn), seconds); + } + int rpcSerializationFlags() override { return RPCSerializationFlags(); } + void requestMempoolTransactions(Notifications& notifications) override + { + LOCK2(::cs_main, ::mempool.cs); + for (const CTxMemPoolEntry& entry : ::mempool.mapTx) { + notifications.TransactionAddedToMempool(entry.GetSharedTx()); + } + } }; - } // namespace std::unique_ptr<Chain> MakeChain() { return MakeUnique<ChainImpl>(); } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 3a54b9164e..0b7249a5ab 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -5,21 +5,56 @@ #ifndef BITCOIN_INTERFACES_CHAIN_H #define BITCOIN_INTERFACES_CHAIN_H -#include <optional.h> +#include <optional.h> // For Optional and nullopt +#include <primitives/transaction.h> // For CTransactionRef #include <memory> +#include <stddef.h> #include <stdint.h> #include <string> #include <vector> class CBlock; +class CFeeRate; +class CRPCCommand; class CScheduler; +class CValidationState; +class Coin; class uint256; +enum class RBFTransactionState; struct CBlockLocator; +struct FeeCalculation; namespace interfaces { -//! Interface for giving wallet processes access to blockchain state. +class Handler; +class Wallet; + +//! Interface giving clients (wallet processes, maybe other analysis tools in +//! the future) ability to access to the chain state, receive notifications, +//! estimate fees, and submit transactions. +//! +//! TODO: Current chain methods are too low level, exposing too much of the +//! internal workings of the bitcoin node, and not being very convenient to use. +//! Chain methods should be cleaned up and simplified over time. Examples: +//! +//! * The Chain::lock() method, which lets clients delay chain tip updates +//! should be removed when clients are able to respond to updates +//! asynchronously +//! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). +//! +//! * The relayTransactions() and submitToMemoryPool() methods could be replaced +//! with a higher-level broadcastTransaction method +//! (https://github.com/bitcoin/bitcoin/pull/14978#issuecomment-459373984). +//! +//! * The initMessages() and loadWallet() methods which the wallet uses to send +//! notifications to the GUI should go away when GUI and wallet can directly +//! communicate with each other without going through the node +//! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). +//! +//! * The handleRpc, registerRpcs, rpcEnableDeprecated methods and other RPC +//! methods can go away if wallets listen for HTTP requests on their own +//! ports instead of registering to handle requests on the node HTTP port. class Chain { public: @@ -64,44 +99,39 @@ public: virtual bool haveBlockOnDisk(int height) = 0; //! Return height of the first block in the chain with timestamp equal - //! or greater than the given time, or nullopt if there is no block with - //! a high enough timestamp. Also return the block hash as an optional - //! output parameter (to avoid the cost of a second lookup in case this - //! information is needed.) - virtual Optional<int> findFirstBlockWithTime(int64_t time, uint256* hash) = 0; - - //! Return height of the first block in the chain with timestamp equal //! or greater than the given time and height equal or greater than the - //! given height, or nullopt if there is no such block. - //! - //! Calling this with height 0 is equivalent to calling - //! findFirstBlockWithTime, but less efficient because it requires a - //! linear instead of a binary search. - virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height) = 0; + //! given height, or nullopt if there is no block with a high enough + //! timestamp and height. Also return the block hash as an optional output parameter + //! (to avoid the cost of a second lookup in case this information is needed.) + virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; //! Return height of last block in the specified range which is pruned, or //! nullopt if no block in the range is pruned. Range is inclusive. virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0; - //! Return height of the highest block on the chain that is an ancestor - //! of the specified block, or nullopt if no common ancestor is found. + //! Return height of the specified block if it is on the chain, otherwise + //! return the height of the highest block on chain that's an ancestor + //! of the specified block, or nullopt if there is no common ancestor. //! Also return the height of the specified block as an optional output //! parameter (to avoid the cost of a second hash lookup in case this //! information is desired). virtual Optional<int> findFork(const uint256& hash, Optional<int>* height) = 0; - //! Return true if block hash points to the current chain tip, or to a - //! possible descendant of the current chain tip that isn't currently - //! connected. - virtual bool isPotentialTip(const uint256& hash) = 0; - //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; - //! Return height of the latest block common to locator and chain, which - //! is guaranteed to be an ancestor of the block used to create the - //! locator. + //! Return height of the highest block on chain in common with the locator, + //! which will either be the original block used to create the locator, + //! or one of its ancestors. virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; + + //! Check if transaction will be final given chain height current time. + virtual bool checkFinalTx(const CTransaction& tx) = 0; + + //! Add transaction to memory pool if the transaction fee is below the + //! amount specified by absurd_fee. Returns false if the transaction + //! could not be added due to the fee or for another reason. + virtual bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) = 0; }; //! Return Lock interface. Chain is locked when this is called, and @@ -124,9 +154,124 @@ public: int64_t* time = nullptr, int64_t* max_time = nullptr) = 0; + //! Look up unspent output information. Returns coins in the mempool and in + //! the current chain UTXO set. Iterates through all the keys in the map and + //! populates the values. + virtual void findCoins(std::map<COutPoint, Coin>& coins) = 0; + //! Estimate fraction of total transactions verified if blocks up to //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + + //! Check if transaction is RBF opt in. + virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; + + //! Check if transaction has descendants in mempool. + virtual bool hasDescendantsInMempool(const uint256& txid) = 0; + + //! Relay transaction. + virtual void relayTransaction(const uint256& txid) = 0; + + //! Calculate mempool ancestor and descendant counts for the given transaction. + virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) = 0; + + //! Check if transaction will pass the mempool's chain limits. + virtual bool checkChainLimits(const CTransactionRef& tx) = 0; + + //! Estimate smart fee. + virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc = nullptr) = 0; + + //! Fee estimator max target. + virtual unsigned int estimateMaxBlocks() = 0; + + //! Mempool minimum fee. + virtual CFeeRate mempoolMinFee() = 0; + + //! Relay current minimum fee (from -minrelaytxfee and -incrementalrelayfee settings). + virtual CFeeRate relayMinFee() = 0; + + //! Relay incremental fee setting (-incrementalrelayfee), reflecting cost of relay. + virtual CFeeRate relayIncrementalFee() = 0; + + //! Relay dust fee setting (-dustrelayfee), reflecting lowest rate it's economical to spend. + virtual CFeeRate relayDustFee() = 0; + + //! Check if pruning is enabled. + virtual bool getPruneMode() = 0; + + //! Check if p2p enabled. + virtual bool p2pEnabled() = 0; + + //! Check if the node is ready to broadcast transactions. + virtual bool isReadyToBroadcast() = 0; + + //! Check if in IBD. + virtual bool isInitialBlockDownload() = 0; + + //! Check if shutdown requested. + virtual bool shutdownRequested() = 0; + + //! Get adjusted time. + virtual int64_t getAdjustedTime() = 0; + + //! Send init message. + virtual void initMessage(const std::string& message) = 0; + + //! Send init warning. + virtual void initWarning(const std::string& message) = 0; + + //! Send init error. + virtual void initError(const std::string& message) = 0; + + //! Send wallet load notification to the GUI. + virtual void loadWallet(std::unique_ptr<Wallet> wallet) = 0; + + //! Send progress indicator. + virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; + + //! Chain notifications. + class Notifications + { + public: + virtual ~Notifications() {} + virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} + virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} + virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} + virtual void BlockDisconnected(const CBlock& block) {} + virtual void UpdatedBlockTip() {} + virtual void ChainStateFlushed(const CBlockLocator& locator) {} + }; + + //! Register handler for notifications. + virtual std::unique_ptr<Handler> handleNotifications(Notifications& notifications) = 0; + + //! Wait for pending notifications to be processed unless block hash points to the current + //! chain tip, or to a possible descendant of the current chain tip that isn't currently + //! connected. + virtual void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) = 0; + + //! Register handler for RPC. Command is not copied, so reference + //! needs to remain valid until Handler is disconnected. + virtual std::unique_ptr<Handler> handleRpc(const CRPCCommand& command) = 0; + + //! Check if deprecated RPC is enabled. + virtual bool rpcEnableDeprecated(const std::string& method) = 0; + + //! Run function after given number of seconds. Cancel any previous calls with same name. + virtual void rpcRunLater(const std::string& name, std::function<void()> fn, int64_t seconds) = 0; + + //! Current RPC serialization flags. + virtual int rpcSerializationFlags() = 0; + + //! Synchronously send TransactionAddedToMempool notifications about all + //! current mempool transactions to the specified handler and return after + //! the last one is sent. These notifications aren't coordinated with async + //! notifications sent by handleNotifications, so out of date async + //! notifications from handleNotifications can arrive during and after + //! synchronous notifications from requestMempoolTransactions. Clients need + //! to be prepared to handle this by ignoring notifications about unknown + //! removed transactions and already added new transactions. + virtual void requestMempoolTransactions(Notifications& notifications) = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 96bde7e9f2..f3ee8fe364 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -20,6 +20,7 @@ #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> +#include <policy/settings.h> #include <primitives/block.h> #include <rpc/server.h> #include <scheduler.h> @@ -206,7 +207,6 @@ public: } } bool getNetworkActive() override { return g_connman && g_connman->GetNetworkActive(); } - CAmount getMaxTxFee() override { return ::maxTxFee; } CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override { FeeCalculation fee_calc; @@ -275,7 +275,7 @@ public: } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::shared_ptr<CWallet> wallet) { fn(MakeWallet(wallet)); })); + return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::unique_ptr<Wallet>& wallet) { fn(std::move(wallet)); })); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 76b93af234..1ccd2a31b7 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -159,9 +159,6 @@ public: //! Get network active. virtual bool getNetworkActive() = 0; - //! Get max tx fee. - virtual CAmount getMaxTxFee() = 0; - //! Estimate smart fee. virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 0dac75834e..b57299d78d 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -29,6 +29,7 @@ #include <wallet/feebumper.h> #include <wallet/fees.h> #include <wallet/rpcwallet.h> +#include <wallet/load.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -47,8 +48,6 @@ public: const CTransaction& get() override { return *m_tx; } - int64_t getVirtualSize() override { return GetVirtualTransactionSize(*m_tx); } - bool commit(WalletValueMap value_map, WalletOrderForm order_form, std::string& reject_reason) override @@ -56,7 +55,7 @@ public: auto locked_chain = m_wallet.chain().lock(); LOCK(m_wallet.cs_wallet); CValidationState state; - if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, g_connman.get(), state)) { + if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, state)) { reject_reason = state.GetRejectReason(); return false; } @@ -99,17 +98,13 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co //! Construct wallet tx status struct. WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - WalletTxStatus result; - auto mi = ::mapBlockIndex.find(wtx.hashBlock); - CBlockIndex* block = mi != ::mapBlockIndex.end() ? mi->second : nullptr; - result.block_height = (block ? block->nHeight : std::numeric_limits<int>::max()); + result.block_height = locked_chain.getBlockHeight(wtx.hashBlock).get_value_or(std::numeric_limits<int>::max()); result.blocks_to_maturity = wtx.GetBlocksToMaturity(locked_chain); result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; - result.is_final = CheckFinalTx(*wtx.tx); + result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); @@ -274,8 +269,13 @@ public: CAmount& new_fee, CMutableTransaction& mtx) override { - return feebumper::CreateTransaction(m_wallet.get(), txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == - feebumper::Result::OK; + if (total_fee > 0) { + return feebumper::CreateTotalBumpTransaction(m_wallet.get(), txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == + feebumper::Result::OK; + } else { + return feebumper::CreateRateBumpTransaction(m_wallet.get(), txid, coin_control, errors, old_fee, new_fee, mtx) == + feebumper::Result::OK; + } } bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(m_wallet.get(), mtx); } bool commitBumpTransaction(const uint256& txid, @@ -364,15 +364,16 @@ public: } WalletBalances getBalances() override { + const auto bal = m_wallet->GetBalance(); WalletBalances result; - result.balance = m_wallet->GetBalance(); - result.unconfirmed_balance = m_wallet->GetUnconfirmedBalance(); - result.immature_balance = m_wallet->GetImmatureBalance(); + result.balance = bal.m_mine_trusted; + result.unconfirmed_balance = bal.m_mine_untrusted_pending; + result.immature_balance = bal.m_mine_immature; result.have_watch_only = m_wallet->HaveWatchOnly(); if (result.have_watch_only) { - result.watch_only_balance = m_wallet->GetBalance(ISMINE_WATCH_ONLY); - result.unconfirmed_watch_only_balance = m_wallet->GetUnconfirmedWatchOnlyBalance(); - result.immature_watch_only_balance = m_wallet->GetImmatureWatchOnlyBalance(); + result.watch_only_balance = bal.m_watchonly_trusted; + result.unconfirmed_watch_only_balance = bal.m_watchonly_untrusted_pending; + result.immature_watch_only_balance = bal.m_watchonly_immature; } return result; } @@ -388,7 +389,7 @@ public: num_blocks = locked_chain->getHeight().get_value_or(-1); return true; } - CAmount getBalance() override { return m_wallet->GetBalance(); } + CAmount getBalance() override { return m_wallet->GetBalance().m_mine_trusted; } CAmount getAvailableBalance(const CCoinControl& coin_control) override { return m_wallet->GetAvailableBalance(&coin_control); @@ -457,7 +458,7 @@ public: { FeeCalculation fee_calc; CAmount result; - result = GetMinimumFee(*m_wallet, tx_bytes, coin_control, ::mempool, ::feeEstimator, &fee_calc); + result = GetMinimumFee(*m_wallet, tx_bytes, coin_control, &fee_calc); if (returned_target) *returned_target = fee_calc.returnedTarget; if (reason) *reason = fee_calc.reason; return result; @@ -468,6 +469,7 @@ public: bool IsWalletFlagSet(uint64_t flag) override { return m_wallet->IsWalletFlagSet(flag); } OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } + CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { RemoveWallet(m_wallet); @@ -514,7 +516,7 @@ public: : m_chain(chain), m_wallet_filenames(std::move(wallet_filenames)) { } - void registerRpcs() override { return RegisterWalletRPCCommands(::tableRPC); } + void registerRpcs() override { return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); } bool verify() override { return VerifyWallets(m_chain, m_wallet_filenames); } bool load() override { return LoadWallets(m_chain, m_wallet_filenames); } void start(CScheduler& scheduler) override { return StartWallets(scheduler); } @@ -524,6 +526,7 @@ public: Chain& m_chain; std::vector<std::string> m_wallet_filenames; + std::vector<std::unique_ptr<Handler>> m_rpc_handlers; }; } // namespace diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index a931e5fafb..7096f54047 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -247,6 +247,9 @@ public: // Get default change type. virtual OutputType getDefaultChangeType() = 0; + //! Get max tx fee. + virtual CAmount getDefaultMaxTxFee() = 0; + // Remove wallet. virtual void remove() = 0; @@ -292,9 +295,6 @@ public: //! Get transaction data. virtual const CTransaction& get() = 0; - //! Get virtual transaction size. - virtual int64_t getVirtualSize() = 0; - //! Send pending transaction and commit to wallet. virtual bool commit(WalletValueMap value_map, WalletOrderForm order_form, diff --git a/src/logging.cpp b/src/logging.cpp index 36cad6573a..3eda4995db 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -4,8 +4,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <logging.h> +#include <util/threadnames.h> #include <util/time.h> +#include <mutex> + const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; BCLog::Logger& LogInstance() @@ -174,7 +177,7 @@ std::vector<CLogCategoryActive> ListActiveLogCategories() return ret; } -std::string BCLog::Logger::LogTimestampStr(const std::string &str) +std::string BCLog::Logger::LogTimestampStr(const std::string& str) { std::string strStamped; @@ -196,21 +199,24 @@ std::string BCLog::Logger::LogTimestampStr(const std::string &str) } else strStamped = str; - if (!str.empty() && str[str.size()-1] == '\n') - m_started_new_line = true; - else - m_started_new_line = false; - return strStamped; } void BCLog::Logger::LogPrintStr(const std::string &str) { - std::string strTimestamped = LogTimestampStr(str); + std::string str_prefixed = str; + + if (m_log_threadnames && m_started_new_line) { + str_prefixed.insert(0, "[" + util::ThreadGetInternalName() + "] "); + } + + str_prefixed = LogTimestampStr(str_prefixed); + + m_started_new_line = !str.empty() && str[str.size()-1] == '\n'; if (m_print_to_console) { // print to console - fwrite(strTimestamped.data(), 1, strTimestamped.size(), stdout); + fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } if (m_print_to_file) { @@ -218,7 +224,7 @@ void BCLog::Logger::LogPrintStr(const std::string &str) // buffer if we haven't opened the log yet if (m_fileout == nullptr) { - m_msgs_before_open.push_back(strTimestamped); + m_msgs_before_open.push_back(str_prefixed); } else { @@ -232,7 +238,7 @@ void BCLog::Logger::LogPrintStr(const std::string &str) m_fileout = new_fileout; } } - FileWriteStr(strTimestamped, m_fileout); + FileWriteStr(str_prefixed, m_fileout); } } } diff --git a/src/logging.h b/src/logging.h index ac9d0dc0c7..e399d4c307 100644 --- a/src/logging.h +++ b/src/logging.h @@ -19,6 +19,7 @@ static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; +static const bool DEFAULT_LOGTHREADNAMES = false; extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; @@ -81,6 +82,7 @@ namespace BCLog { bool m_log_timestamps = DEFAULT_LOGTIMESTAMPS; bool m_log_time_micros = DEFAULT_LOGTIMEMICROS; + bool m_log_threadnames = DEFAULT_LOGTHREADNAMES; fs::path m_file_path; std::atomic<bool> m_reopen_file{false}; diff --git a/src/miner.cpp b/src/miner.cpp index 80a2f8f018..6a88e8321d 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -23,6 +23,7 @@ #include <timedata.h> #include <util/moneystr.h> #include <util/system.h> +#include <util/validation.h> #include <validationinterface.h> #include <algorithm> diff --git a/src/net.cpp b/src/net.cpp index 87f1ef0577..1335804b06 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1765,9 +1765,15 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) addr = addrman.Select(fFeeler); } - // if we selected an invalid address, restart - if (!addr.IsValid() || setConnected.count(addr.GetGroup()) || IsLocal(addr)) + // Require outbound connections, other than feelers, to be to distinct network groups + if (!fFeeler && setConnected.count(addr.GetGroup())) { break; + } + + // if we selected an invalid or local address, restart + if (!addr.IsValid() || IsLocal(addr)) { + break; + } // If we didn't find an appropriate destination after trying 100 addresses fetched from addrman, // stop this loop, and let the outer loop run again (which sleeps, adds seed nodes, recalculates @@ -2621,7 +2627,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn { hSocket = hSocketIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; - strSubVer = ""; hashContinue = uint256(); filterInventoryKnown.reset(); pfilter = MakeUnique<CBloomFilter>(); @@ -53,7 +53,7 @@ static const unsigned int MAX_LOCATOR_SZ = 101; static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ static const unsigned int MAX_PROTOCOL_MESSAGE_LENGTH = 4 * 1000 * 1000; -/** Maximum length of strSubVer in `version` message */ +/** Maximum length of the user agent string in `version` message */ static const unsigned int MAX_SUBVERSION_LENGTH = 256; /** Maximum number of automatic outgoing nodes */ static const int MAX_OUTBOUND_CONNECTIONS = 8; @@ -650,12 +650,12 @@ public: // Bind address of our side of the connection const CAddress addrBind; std::atomic<int> nVersion{0}; - // strSubVer is whatever byte array we read from the wire. However, this field is intended - // to be printed out, displayed to humans in various forms and so on. So we sanitize it and - // store the sanitized version in cleanSubVer. The original should be used when dealing with - // the network or wire types and the cleaned string used when displayed or logged. - std::string strSubVer GUARDED_BY(cs_SubVer), cleanSubVer GUARDED_BY(cs_SubVer); - CCriticalSection cs_SubVer; // used for both cleanSubVer and strSubVer + RecursiveMutex cs_SubVer; + /** + * cleanSubVer is a sanitized string of the user agent byte array we read + * from the wire. This cleaned string can safely be logged or displayed. + */ + std::string cleanSubVer GUARDED_BY(cs_SubVer){}; bool m_prefer_evict{false}; // This peer is preferred for eviction. bool fWhitelisted{false}; // This peer can bypass DoS banning. bool fFeeler{false}; // If true this node is being used as a short lived feeler. @@ -739,6 +739,8 @@ public: CAmount lastSentFeeFilter{0}; int64_t nextSendTimeFeeFilter{0}; + std::set<uint256> orphan_work_set; + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); CNode(const CNode&) = delete; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 18dd2f010c..71ebd72b83 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -29,6 +29,7 @@ #include <util/system.h> #include <util/moneystr.h> #include <util/strencodings.h> +#include <util/validation.h> #include <memory> @@ -175,8 +176,6 @@ namespace { /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration GUARDED_BY(cs_main); - std::atomic<int64_t> nTimeBestReceived(0); // Used only to inform the wallet of when we last received a block - struct IteratorComparator { template<typename I> @@ -352,7 +351,16 @@ struct CNodeState { TxDownloadState m_tx_download; - CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { + //! Whether this peer is an inbound connection + bool m_is_inbound; + + //! Whether this peer is a manual connection + bool m_is_manual_connection; + + CNodeState(CAddress addrIn, std::string addrNameIn, bool is_inbound, bool is_manual) : + address(addrIn), name(std::move(addrNameIn)), m_is_inbound(is_inbound), + m_is_manual_connection (is_manual) + { fCurrentlyConnected = false; nMisbehavior = 0; fShouldBan = false; @@ -748,7 +756,7 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { NodeId nodeid = pnode->GetId(); { LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->fInbound, pnode->m_manual_connection)); } if(!pnode->fInbound) PushNodeVersion(pnode, connman, GetTime()); @@ -960,6 +968,90 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIV LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d)%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); } +/** + * Returns true if the given validation state result may result in a peer + * banning/disconnecting us. We use this to determine which unaccepted + * transactions from a whitelisted peer that we can safely relay. + */ +static bool TxRelayMayResultInDisconnect(const CValidationState& state) +{ + assert(IsTransactionReason(state.GetReason())); + return state.GetReason() == ValidationInvalidReason::CONSENSUS; +} + +/** + * Potentially ban a node based on the contents of a CValidationState object + * + * @param[in] via_compact_block: this bool is passed in because net_processing should + * punish peers differently depending on whether the data was provided in a compact + * block message or not. If the compact block had a valid header, but contained invalid + * txs, the peer should not be punished. See BIP 152. + * + * @return Returns true if the peer was punished (probably disconnected) + * + * Changes here may need to be reflected in TxRelayMayResultInDisconnect(). + */ +static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool via_compact_block, const std::string& message = "") { + switch (state.GetReason()) { + case ValidationInvalidReason::NONE: + break; + // The node is providing invalid data: + case ValidationInvalidReason::CONSENSUS: + case ValidationInvalidReason::BLOCK_MUTATED: + if (!via_compact_block) { + LOCK(cs_main); + Misbehaving(nodeid, 100, message); + return true; + } + break; + case ValidationInvalidReason::CACHED_INVALID: + { + LOCK(cs_main); + CNodeState *node_state = State(nodeid); + if (node_state == nullptr) { + break; + } + + // Ban outbound (but not inbound) peers if on an invalid chain. + // Exempt HB compact block peers and manual connections. + if (!via_compact_block && !node_state->m_is_inbound && !node_state->m_is_manual_connection) { + Misbehaving(nodeid, 100, message); + return true; + } + break; + } + case ValidationInvalidReason::BLOCK_INVALID_HEADER: + case ValidationInvalidReason::BLOCK_CHECKPOINT: + case ValidationInvalidReason::BLOCK_INVALID_PREV: + { + LOCK(cs_main); + Misbehaving(nodeid, 100, message); + } + return true; + // Conflicting (but not necessarily invalid) data or different policy: + case ValidationInvalidReason::BLOCK_MISSING_PREV: + { + // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) + LOCK(cs_main); + Misbehaving(nodeid, 10, message); + } + return true; + case ValidationInvalidReason::RECENT_CONSENSUS_CHANGE: + case ValidationInvalidReason::BLOCK_TIME_FUTURE: + case ValidationInvalidReason::TX_NOT_STANDARD: + case ValidationInvalidReason::TX_MISSING_INPUTS: + case ValidationInvalidReason::TX_PREMATURE_SPEND: + case ValidationInvalidReason::TX_WITNESS_MUTATED: + case ValidationInvalidReason::TX_CONFLICT: + case ValidationInvalidReason::TX_MEMPOOL_POLICY: + break; + } + if (message != "") { + LogPrint(BCLog::NET, "peer=%d: %s\n", nodeid, message); + } + return false; +} + @@ -1121,8 +1213,6 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB }); connman->WakeMessageHandler(); } - - nTimeBestReceived = GetTime(); } /** @@ -1135,14 +1225,12 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta const uint256 hash(block.GetHash()); std::map<uint256, std::pair<NodeId, bool>>::iterator it = mapBlockSource.find(hash); - int nDoS = 0; - if (state.IsInvalid(nDoS)) { + if (state.IsInvalid()) { // Don't send reject message with code 0 or an internal reject code. if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; State(it->second.first)->rejects.push_back(reject); - if (nDoS > 0 && it->second.second) - Misbehaving(it->second.first, nDoS); + MaybePunishNode(/*nodeid=*/ it->second.first, state, /*via_compact_block=*/ !it->second.second); } } // Check that: @@ -1492,7 +1580,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool punish_duplicate_invalid) +bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); size_t nCount = headers.size(); @@ -1554,48 +1642,8 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve CValidationState state; CBlockHeader first_invalid_header; if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - LOCK(cs_main); - if (nDoS > 0) { - Misbehaving(pfrom->GetId(), nDoS, "invalid header received"); - } else { - LogPrint(BCLog::NET, "peer=%d: invalid header received\n", pfrom->GetId()); - } - if (punish_duplicate_invalid && LookupBlockIndex(first_invalid_header.GetHash())) { - // Goal: don't allow outbound peers to use up our outbound - // connection slots if they are on incompatible chains. - // - // We ask the caller to set punish_invalid appropriately based - // on the peer and the method of header delivery (compact - // blocks are allowed to be invalid in some circumstances, - // under BIP 152). - // Here, we try to detect the narrow situation that we have a - // valid block header (ie it was valid at the time the header - // was received, and hence stored in mapBlockIndex) but know the - // block is invalid, and that a peer has announced that same - // block as being on its active chain. - // Disconnect the peer in such a situation. - // - // Note: if the header that is invalid was not accepted to our - // mapBlockIndex at all, that may also be grounds for - // disconnecting the peer, as the chain they are on is likely - // to be incompatible. However, there is a circumstance where - // that does not hold: if the header's timestamp is more than - // 2 hours ahead of our current time. In that case, the header - // may become valid in the future, and we don't want to - // disconnect a peer merely for serving us one too-far-ahead - // block header, to prevent an attacker from splitting the - // network by mining a block right at the 2 hour boundary. - // - // TODO: update the DoS logic (or, rather, rewrite the - // DoS-interface between validation and net_processing) so that - // the interface is cleaner, and so that we disconnect on all the - // reasons that a peer's headers chain is incompatible - // with ours (eg block->nVersion softforks, MTP violations, - // etc), and not just the duplicate-invalid case. - pfrom->fDisconnect = true; - } + if (state.IsInvalid()) { + MaybePunishNode(pfrom->GetId(), state, via_compact_block, "invalid header received"); return false; } } @@ -1713,6 +1761,68 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve return true; } +void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +{ + AssertLockHeld(cs_main); + AssertLockHeld(g_cs_orphans); + std::set<NodeId> setMisbehaving; + bool done = false; + while (!done && !orphan_work_set.empty()) { + const uint256 orphanHash = *orphan_work_set.begin(); + orphan_work_set.erase(orphan_work_set.begin()); + + auto orphan_it = mapOrphanTransactions.find(orphanHash); + if (orphan_it == mapOrphanTransactions.end()) continue; + + const CTransactionRef porphanTx = orphan_it->second.tx; + const CTransaction& orphanTx = *porphanTx; + NodeId fromPeer = orphan_it->second.fromPeer; + bool fMissingInputs2 = false; + // Use a new CValidationState because orphans come from different peers (and we call + // MaybePunishNode based on the source peer from the orphan map, not based on the peer + // that relayed the previous transaction). + CValidationState orphan_state; + + if (setMisbehaving.count(fromPeer)) continue; + if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &fMissingInputs2, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); + RelayTransaction(orphanTx, connman); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + orphan_work_set.insert(elem->first); + } + } + } + EraseOrphanTx(orphanHash); + done = true; + } else if (!fMissingInputs2) { + if (orphan_state.IsInvalid()) { + // Punish peer that gave us an invalid orphan tx + if (MaybePunishNode(fromPeer, orphan_state, /*via_compact_block*/ false)) { + setMisbehaving.insert(fromPeer); + } + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); + } + // Has inputs but not accepted to mempool + // Probably non-standard or insufficient fee + LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); + assert(IsTransactionReason(orphan_state.GetReason())); + if (!orphanTx.HasWitness() && orphan_state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + assert(recentRejects); + recentRejects->insert(orphanHash); + } + EraseOrphanTx(orphanHash); + done = true; + } + mempool.check(pcoinsTip.get()); + } +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc, bool enable_bip61) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -1782,7 +1892,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ServiceFlags nServices; int nVersion; int nSendVersion; - std::string strSubVer; std::string cleanSubVer; int nStartingHeight = -1; bool fRelay = true; @@ -1819,6 +1928,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!vRecv.empty()) vRecv >> addrFrom >> nNonce; if (!vRecv.empty()) { + std::string strSubVer; vRecv >> LIMITED_STRING(strSubVer, MAX_SUBVERSION_LENGTH); cleanSubVer = SanitizeString(strSubVer); } @@ -1850,7 +1960,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->SetAddrLocal(addrMe); { LOCK(pfrom->cs_SubVer); - pfrom->strSubVer = strSubVer; pfrom->cleanSubVer = cleanSubVer; } pfrom->nStartingHeight = nStartingHeight; @@ -2022,6 +2131,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); + if (g_banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -2341,8 +2451,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - std::deque<COutPoint> vWorkQueue; - std::vector<uint256> vEraseQueue; CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; @@ -2367,7 +2475,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.check(pcoinsTip.get()); RelayTransaction(tx, connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { - vWorkQueue.emplace_back(inv.hash, i); + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + pfrom->orphan_work_set.insert(elem->first); + } + } } pfrom->nLastTXTime = GetTime(); @@ -2378,65 +2491,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - std::set<NodeId> setMisbehaving; - while (!vWorkQueue.empty()) { - auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); - vWorkQueue.pop_front(); - if (itByPrev == mapOrphanTransactionsByPrev.end()) - continue; - for (auto mi = itByPrev->second.begin(); - mi != itByPrev->second.end(); - ++mi) - { - const CTransactionRef& porphanTx = (*mi)->second.tx; - const CTransaction& orphanTx = *porphanTx; - const uint256& orphanHash = orphanTx.GetHash(); - NodeId fromPeer = (*mi)->second.fromPeer; - bool fMissingInputs2 = false; - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan - // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get - // anyone relaying LegitTxX banned) - CValidationState stateDummy; - - - if (setMisbehaving.count(fromPeer)) - continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, &fMissingInputs2, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, connman); - for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { - vWorkQueue.emplace_back(orphanHash, i); - } - vEraseQueue.push_back(orphanHash); - } - else if (!fMissingInputs2) - { - int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) - { - // Punish peer that gave us an invalid orphan tx - Misbehaving(fromPeer, nDos); - setMisbehaving.insert(fromPeer); - LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); - } - // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee - LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - vEraseQueue.push_back(orphanHash); - if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. - assert(recentRejects); - recentRejects->insert(orphanHash); - } - } - mempool.check(pcoinsTip.get()); - } - } - - for (const uint256& hash : vEraseQueue) - EraseOrphanTx(hash); + ProcessOrphanTx(connman, pfrom->orphan_work_set, lRemovedTxn); } else if (fMissingInputs) { @@ -2471,7 +2526,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(tx.GetHash()); } } else { - if (!tx.HasWitness() && !state.CorruptionPossible()) { + assert(IsTransactionReason(state.GetReason())); + if (!tx.HasWitness() && state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. @@ -2490,15 +2546,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // to policy, allowing the node to function as a gateway for // nodes hidden behind it. // - // Never relay transactions that we would assign a non-zero DoS - // score for, as we expect peers to do the same with us in that - // case. - int nDoS = 0; - if (!state.IsInvalid(nDoS) || nDoS == 0) { + // Never relay transactions that might result in being + // disconnected (or banned). + if (state.IsInvalid() && TxRelayMayResultInDisconnect(state)) { + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); + } else { LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); RelayTransaction(tx, connman); - } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } } } @@ -2523,8 +2577,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // peer simply for relaying a tx that our recentRejects has caught, // regardless of false positives. - int nDoS = 0; - if (state.IsInvalid(nDoS)) + if (state.IsInvalid()) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom->GetId(), @@ -2533,15 +2586,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); } - if (nDoS > 0) { - Misbehaving(pfrom->GetId(), nDoS); - } + MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ false); } return true; } - if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::CMPCTBLOCK) { + // Ignore cmpctblock received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom->GetId()); + return true; + } + CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; @@ -2565,14 +2622,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr const CBlockIndex *pindex = nullptr; CValidationState state; if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS, strprintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId())); - } else { - LogPrint(BCLog::NET, "Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); - } + if (state.IsInvalid()) { + MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return true; } } @@ -2722,7 +2773,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be banned. - return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*punish_duplicate_invalid=*/false); + return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -2761,8 +2812,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::BLOCKTXN) { + // Ignore blocktxn received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom->GetId()); + return true; + } + BlockTransactions resp; vRecv >> resp; @@ -2836,8 +2893,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing + if (strCommand == NetMsgType::HEADERS) { + // Ignore headers received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom->GetId()); + return true; + } + std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -2853,16 +2916,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - // Headers received via a HEADERS message should be valid, and reflect - // the chain the peer is on. If we receive a known-invalid header, - // disconnect the peer if it is using one of our outbound connection - // slots. - bool should_punish = !pfrom->fInbound && !pfrom->m_manual_connection; - return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); + return ProcessHeadersMessage(pfrom, connman, headers, chainparams, /*via_compact_block=*/false); } - if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::BLOCK) { + // Ignore block received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom->GetId()); + return true; + } + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; @@ -2912,8 +2976,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->vAddrToSend.clear(); std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; - for (const CAddress &addr : vAddr) - pfrom->PushAddress(addr, insecure_rand); + for (const CAddress &addr : vAddr) { + if (!g_banman->IsBanned(addr)) { + pfrom->PushAddress(addr, insecure_rand); + } + } return true; } @@ -3141,11 +3208,21 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!pfrom->vRecvGetData.empty()) ProcessGetData(pfrom, chainparams, connman, interruptMsgProc); + if (!pfrom->orphan_work_set.empty()) { + std::list<CTransactionRef> removed_txn; + LOCK2(cs_main, g_cs_orphans); + ProcessOrphanTx(connman, pfrom->orphan_work_set, removed_txn); + for (const CTransactionRef& removedTx : removed_txn) { + AddToCompactExtraTransactions(removedTx); + } + } + if (pfrom->fDisconnect) return false; // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; + if (!pfrom->orphan_work_set.empty()) return true; // Don't bother if send buffer is too full to respond anyway if (pfrom->fPauseSend) @@ -3507,14 +3584,6 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } - // Resend wallet transactions that haven't gotten in a block yet - // Except during reindex, importing and IBD, when old wallet - // transactions become unconfirmed and spams other nodes. - if (!fReindex && !fImporting && !IsInitialBlockDownload()) - { - GetMainSignals().Broadcast(nTimeBestReceived, connman); - } - // // Try sending block announcements via headers // diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 58e45c2c02..6ee2d8a4b3 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -14,6 +14,11 @@ static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; // 0xFD + sha256("bitcoin")[0:5] static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; +/** + * Construct an unspecified IPv6 network address (::/128). + * + * @note This address is considered invalid by CNetAddr::IsValid() + */ CNetAddr::CNetAddr() { memset(ip, 0, sizeof(ip)); @@ -40,6 +45,20 @@ void CNetAddr::SetRaw(Network network, const uint8_t *ip_in) } } +/** + * Try to make this a dummy address that maps the specified name into IPv6 like + * so: (0xFD + %sha256("bitcoin")[0:5]) + %sha256(name)[0:10]. Such dummy + * addresses have a prefix of fd6b:88c0:8724::/48 and are guaranteed to not be + * publicly routable as it falls under RFC4193's fc00::/7 subnet allocated to + * unique-local addresses. + * + * CAddrMan uses these fake addresses to keep track of which DNS seeds were + * used. + * + * @returns Whether or not the operation was successful. + * + * @see CNetAddr::IsInternal(), CNetAddr::IsRFC4193() + */ bool CNetAddr::SetInternal(const std::string &name) { if (name.empty()) { @@ -52,6 +71,16 @@ bool CNetAddr::SetInternal(const std::string &name) return true; } +/** + * Try to make this a dummy address that maps the specified onion address into + * IPv6 using OnionCat's range and encoding. Such dummy addresses have a prefix + * of fd87:d87e:eb43::/48 and are guaranteed to not be publicly routable as they + * fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. + * + * @returns Whether or not the operation was successful. + * + * @see CNetAddr::IsTor(), CNetAddr::IsRFC4193() + */ bool CNetAddr::SetSpecial(const std::string &strName) { if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { @@ -175,6 +204,12 @@ bool CNetAddr::IsRFC4843() const return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10); } +/** + * @returns Whether or not this is a dummy address that maps an onion address + * into IPv6. + * + * @see CNetAddr::SetSpecial(const std::string &) + */ bool CNetAddr::IsTor() const { return (memcmp(ip, pchOnionCat, sizeof(pchOnionCat)) == 0); @@ -194,6 +229,16 @@ bool CNetAddr::IsLocal() const return false; } +/** + * @returns Whether or not this network address is a valid address that @a could + * be used to refer to an actual host. + * + * @note A valid address may or may not be publicly routable on the global + * internet. As in, the set of valid addreses is a superset of the set of + * publicly routable addresses. + * + * @see CNetAddr::IsRoutable() + */ bool CNetAddr::IsValid() const { // Cleanup 3-byte shifted addresses caused by garbage in size field @@ -233,11 +278,25 @@ bool CNetAddr::IsValid() const return true; } +/** + * @returns Whether or not this network address is publicly routable on the + * global internet. + * + * @note A routable address is always valid. As in, the set of routable addreses + * is a subset of the set of valid addresses. + * + * @see CNetAddr::IsValid() + */ bool CNetAddr::IsRoutable() const { return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsLocal() || IsInternal()); } +/** + * @returns Whether or not this is a dummy address that maps a name into IPv6. + * + * @see CNetAddr::SetInternal(const std::string &) + */ bool CNetAddr::IsInternal() const { return memcmp(ip, g_internal_prefix, sizeof(g_internal_prefix)) == 0; @@ -299,6 +358,16 @@ bool operator<(const CNetAddr& a, const CNetAddr& b) return (memcmp(a.ip, b.ip, 16) < 0); } +/** + * Try to get our IPv4 address. + * + * @param[out] pipv4Addr The in_addr struct to which to copy. + * + * @returns Whether or not the operation was successful, in particular, whether + * or not our address was an IPv4 address. + * + * @see CNetAddr::IsIPv4() + */ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const { if (!IsIPv4()) @@ -307,6 +376,16 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const return true; } +/** + * Try to get our IPv6 address. + * + * @param[out] pipv6Addr The in6_addr struct to which to copy. + * + * @returns Whether or not the operation was successful, in particular, whether + * or not our address was an IPv6 address. + * + * @see CNetAddr::IsIPv6() + */ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const { if (!IsIPv6()) { @@ -316,8 +395,16 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } -// get canonical identifier of an address' group -// no two connections will be attempted to addresses with the same group +/** + * Get the canonical identifier of our network group + * + * The groups are assigned in a way where it should be costly for an attacker to + * obtain addresses with many different group identifiers, even if it is cheap + * to obtain addresses with the same identifier. + * + * @note No two connections will be attempted to addresses with the same network + * group. + */ std::vector<unsigned char> CNetAddr::GetGroup() const { std::vector<unsigned char> vchRet; @@ -379,12 +466,15 @@ std::vector<unsigned char> CNetAddr::GetGroup() const nBits = 32; vchRet.push_back(nClass); + + // push our ip onto vchRet byte by byte... while (nBits >= 8) { vchRet.push_back(GetByte(15 - nStartByte)); nStartByte++; nBits -= 8; } + // ...for the last byte, push nBits and for the rest of the byte push 1's if (nBits > 0) vchRet.push_back(GetByte(15 - nStartByte) | ((1 << (8 - nBits)) - 1)); @@ -526,6 +616,18 @@ bool operator<(const CService& a, const CService& b) return static_cast<CNetAddr>(a) < static_cast<CNetAddr>(b) || (static_cast<CNetAddr>(a) == static_cast<CNetAddr>(b) && a.port < b.port); } +/** + * Obtain the IPv4/6 socket address this represents. + * + * @param[out] paddr The obtained socket address. + * @param[in,out] addrlen The size, in bytes, of the address structure pointed + * to by paddr. The value that's pointed to by this + * parameter might change after calling this function if + * the size of the corresponding address structure + * changed. + * + * @returns Whether or not the operation was successful. + */ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const { if (IsIPv4()) { @@ -556,13 +658,16 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const return false; } +/** + * @returns An identifier unique to this service's address and port number. + */ std::vector<unsigned char> CService::GetKey() const { std::vector<unsigned char> vKey; vKey.resize(18); memcpy(vKey.data(), ip, 16); - vKey[16] = port / 0x100; - vKey[17] = port & 0x0FF; + vKey[16] = port / 0x100; // most significant byte of our port + vKey[17] = port & 0x0FF; // least significant byte of our port return vKey; } @@ -641,6 +746,10 @@ CSubNet::CSubNet(const CNetAddr &addr): network = addr; } +/** + * @returns True if this subnet is valid, the specified address is valid, and + * the specified address belongs in this subnet. + */ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid()) @@ -651,6 +760,10 @@ bool CSubNet::Match(const CNetAddr &addr) const return true; } +/** + * @returns The number of 1-bits in the prefix of the specified subnet mask. If + * the specified subnet mask is not a valid one, -1. + */ static inline int NetmaskBits(uint8_t x) { switch(x) { diff --git a/src/netaddress.h b/src/netaddress.h index ca435d17dc..8230e40606 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -48,10 +48,6 @@ class CNetAddr void SetRaw(Network network, const uint8_t *data); public: - /** - * Transform an arbitrary string into a non-routable ipv6 address. - * Useful for mapping resolved addresses back to their source. - */ bool SetInternal(const std::string& name); bool SetSpecial(const std::string &strName); // for Tor addresses @@ -69,8 +65,8 @@ class CNetAddr bool IsRFC4380() const; // IPv6 Teredo tunnelling (2001::/32) bool IsRFC4843() const; // IPv6 ORCHID (2001:10::/28) bool IsRFC4862() const; // IPv6 autoconfig (FE80::/64) - bool IsRFC6052() const; // IPv6 well-known prefix (64:FF9B::/96) - bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) + bool IsRFC6052() const; // IPv6 well-known prefix for IPv4-embedded address (64:FF9B::/96) + bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) (actually defined in RFC2765) bool IsTor() const; bool IsLocal() const; bool IsRoutable() const; diff --git a/src/node/README.md b/src/node/README.md new file mode 100644 index 0000000000..e99a717534 --- /dev/null +++ b/src/node/README.md @@ -0,0 +1,22 @@ +# src/node/ + +The [`src/node/`](./) directory contains code that needs to access node state +(state in `CChain`, `CBlockIndex`, `CCoinsView`, `CTxMemPool`, and similar +classes). + +Code in [`src/node/`](./) is meant to be segregated from code in +[`src/wallet/`](../wallet/) and [`src/qt/`](../qt/), to ensure wallet and GUI +code changes don't interfere with node operation, to allow wallet and GUI code +to run in separate processes, and to perhaps eventually allow wallet and GUI +code to be maintained in separate source repositories. + +As a rule of thumb, code in one of the [`src/node/`](./), +[`src/wallet/`](../wallet/), or [`src/qt/`](../qt/) directories should avoid +calling code in the other directories directly, and only invoke it indirectly +through the more limited [`src/interfaces/`](../interfaces/) classes. + +The [`src/node/`](./) directory is a new directory introduced in +[#14978](https://github.com/bitcoin/bitcoin/pull/14978) and at the moment is +sparsely populated. Eventually more substantial files like +[`src/validation.cpp`](../validation.cpp) and +[`src/txmempool.cpp`](../txmempool.cpp) might be moved there. diff --git a/src/node/coin.cpp b/src/node/coin.cpp new file mode 100644 index 0000000000..bb98e63f3a --- /dev/null +++ b/src/node/coin.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/coin.h> + +#include <txmempool.h> +#include <validation.h> + +void FindCoins(std::map<COutPoint, Coin>& coins) +{ + LOCK2(cs_main, ::mempool.cs); + assert(pcoinsTip); + CCoinsViewCache& chain_view = *::pcoinsTip; + CCoinsViewMemPool mempool_view(&chain_view, ::mempool); + for (auto& coin : coins) { + if (!mempool_view.GetCoin(coin.first, coin.second)) { + // Either the coin is not in the CCoinsViewCache or is spent. Clear it. + coin.second.Clear(); + } + } +} diff --git a/src/node/coin.h b/src/node/coin.h new file mode 100644 index 0000000000..eb95b75cfb --- /dev/null +++ b/src/node/coin.h @@ -0,0 +1,22 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_COIN_H +#define BITCOIN_NODE_COIN_H + +#include <map> + +class COutPoint; +class Coin; + +/** + * Look up unspent output information. Returns coins in the mempool and in the + * current chain UTXO set. Iterates through all the keys in the map and + * populates the values. + * + * @param[in,out] coins map to fill + */ +void FindCoins(std::map<COutPoint, Coin>& coins); + +#endif // BITCOIN_NODE_COIN_H diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp new file mode 100644 index 0000000000..12559c5a5f --- /dev/null +++ b/src/node/psbt.cpp @@ -0,0 +1,134 @@ +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <coins.h> +#include <consensus/tx_verify.h> +#include <node/psbt.h> +#include <policy/policy.h> +#include <policy/settings.h> + +#include <numeric> + +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) +{ + // Go through each input and build status + PSBTAnalysis result; + + bool calc_fee = true; + bool all_final = true; + bool only_missing_sigs = true; + bool only_missing_final = false; + CAmount in_amt = 0; + + result.inputs.resize(psbtx.tx->vin.size()); + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + PSBTInputAnalysis& input_analysis = result.inputs[i]; + + // Check for a UTXO + CTxOut utxo; + if (psbtx.GetInputUTXO(utxo, i)) { + in_amt += utxo.nValue; + input_analysis.has_utxo = true; + } else { + input_analysis.has_utxo = false; + input_analysis.is_final = false; + input_analysis.next = PSBTRole::UPDATER; + calc_fee = false; + } + + // Check if it is final + if (!utxo.IsNull() && !PSBTInputSigned(input)) { + input_analysis.is_final = false; + all_final = false; + + // Figure out what is missing + SignatureData outdata; + bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); + + // Things are missing + if (!complete) { + input_analysis.missing_pubkeys = outdata.missing_pubkeys; + input_analysis.missing_redeem_script = outdata.missing_redeem_script; + input_analysis.missing_witness_script = outdata.missing_witness_script; + input_analysis.missing_sigs = outdata.missing_sigs; + + // If we are only missing signatures and nothing else, then next is signer + if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { + input_analysis.next = PSBTRole::SIGNER; + } else { + only_missing_sigs = false; + input_analysis.next = PSBTRole::UPDATER; + } + } else { + only_missing_final = true; + input_analysis.next = PSBTRole::FINALIZER; + } + } else if (!utxo.IsNull()){ + input_analysis.is_final = true; + } + } + + if (all_final) { + only_missing_sigs = false; + result.next = PSBTRole::EXTRACTOR; + } + if (calc_fee) { + // Get the output amount + CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), + [](CAmount a, const CTxOut& b) { + return a += b.nValue; + } + ); + + // Get the fee + CAmount fee = in_amt - out_amt; + result.fee = fee; + + // Estimate the size + CMutableTransaction mtx(*psbtx.tx); + CCoinsView view_dummy; + CCoinsViewCache view(&view_dummy); + bool success = true; + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + Coin newcoin; + + if (!SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true) || !psbtx.GetInputUTXO(newcoin.out, i)) { + success = false; + break; + } else { + mtx.vin[i].scriptSig = input.final_script_sig; + mtx.vin[i].scriptWitness = input.final_script_witness; + newcoin.nHeight = 1; + view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); + } + } + + if (success) { + CTransaction ctx = CTransaction(mtx); + size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); + result.estimated_vsize = size; + // Estimate fee rate + CFeeRate feerate(fee, size); + result.estimated_feerate = feerate; + } + + if (only_missing_sigs) { + result.next = PSBTRole::SIGNER; + } else if (only_missing_final) { + result.next = PSBTRole::FINALIZER; + } else if (all_final) { + result.next = PSBTRole::EXTRACTOR; + } else { + result.next = PSBTRole::UPDATER; + } + } else { + result.next = PSBTRole::UPDATER; + } + + return result; +} diff --git a/src/node/psbt.h b/src/node/psbt.h new file mode 100644 index 0000000000..e04366a20f --- /dev/null +++ b/src/node/psbt.h @@ -0,0 +1,43 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_PSBT_H +#define BITCOIN_NODE_PSBT_H + +#include <psbt.h> + +/** + * Holds an analysis of one input from a PSBT + */ +struct PSBTInputAnalysis { + bool has_utxo; //!< Whether we have UTXO information for this input + bool is_final; //!< Whether the input has all required information including signatures + PSBTRole next; //!< Which of the BIP 174 roles needs to handle this input next + + std::vector<CKeyID> missing_pubkeys; //!< Pubkeys whose BIP32 derivation path is missing + std::vector<CKeyID> missing_sigs; //!< Pubkeys whose signatures are missing + uint160 missing_redeem_script; //!< Hash160 of redeem script, if missing + uint256 missing_witness_script; //!< SHA256 of witness script, if missing +}; + +/** + * Holds the results of AnalyzePSBT (miscellaneous information about a PSBT) + */ +struct PSBTAnalysis { + Optional<size_t> estimated_vsize; //!< Estimated weight of the transaction + Optional<CFeeRate> estimated_feerate; //!< Estimated feerate (fee / weight) of the transaction + Optional<CAmount> fee; //!< Amount of fee being paid by the transaction + std::vector<PSBTInputAnalysis> inputs; //!< More information about the individual inputs of the transaction + PSBTRole next; //!< Which of the BIP 174 roles needs to handle the transaction next +}; + +/** + * Provides helpful miscellaneous information about where a PSBT is in the signing workflow. + * + * @param[in] psbtx the PSBT to analyze + * @return A PSBTAnalysis with information about the provided PSBT. + */ +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); + +#endif // BITCOIN_NODE_PSBT_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 7b9b4310e7..5ffb15ed3c 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -6,38 +6,13 @@ #include <consensus/validation.h> #include <net.h> #include <txmempool.h> +#include <util/validation.h> #include <validation.h> #include <validationinterface.h> #include <node/transaction.h> #include <future> -std::string TransactionErrorString(const TransactionError err) -{ - switch (err) { - case TransactionError::OK: - return "No error"; - case TransactionError::MISSING_INPUTS: - return "Missing inputs"; - case TransactionError::ALREADY_IN_CHAIN: - return "Transaction already in block chain"; - case TransactionError::P2P_DISABLED: - return "Peer-to-peer functionality missing or disabled"; - case TransactionError::MEMPOOL_REJECTED: - return "Transaction rejected by AcceptToMemoryPool"; - case TransactionError::MEMPOOL_ERROR: - return "AcceptToMemoryPool failed"; - case TransactionError::INVALID_PSBT: - return "PSBT is not sane"; - case TransactionError::PSBT_MISMATCH: - return "PSBTs not compatible (different transactions)"; - case TransactionError::SIGHASH_MISMATCH: - return "Specified sighash value does not match existing value"; - // no default case, so the compiler can warn about missing cases - } - assert(false); -} - TransactionError BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, std::string& err_string, const CAmount& highfee) { std::promise<void> promise; diff --git a/src/node/transaction.h b/src/node/transaction.h index 3457ececa4..51033f94e5 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -8,20 +8,7 @@ #include <attributes.h> #include <primitives/transaction.h> #include <uint256.h> - -enum class TransactionError { - OK, //!< No error - MISSING_INPUTS, - ALREADY_IN_CHAIN, - P2P_DISABLED, - MEMPOOL_REJECTED, - MEMPOOL_ERROR, - INVALID_PSBT, - PSBT_MISMATCH, - SIGHASH_MISMATCH, -}; - -std::string TransactionErrorString(const TransactionError error); +#include <util/error.h> /** * Broadcast a transaction diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index c49b9fa36b..524afd014e 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -27,40 +27,6 @@ std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) { return horizon_string->second; } -std::string StringForFeeReason(FeeReason reason) { - static const std::map<FeeReason, std::string> fee_reason_strings = { - {FeeReason::NONE, "None"}, - {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, - {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, - {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, - {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, - {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, - {FeeReason::PAYTXFEE, "PayTxFee set"}, - {FeeReason::FALLBACK, "Fallback fee"}, - {FeeReason::REQUIRED, "Minimum Required Fee"}, - {FeeReason::MAXTXFEE, "MaxTxFee limit"} - }; - auto reason_string = fee_reason_strings.find(reason); - - if (reason_string == fee_reason_strings.end()) return "Unknown"; - - return reason_string->second; -} - -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) { - static const std::map<std::string, FeeEstimateMode> fee_modes = { - {"UNSET", FeeEstimateMode::UNSET}, - {"ECONOMICAL", FeeEstimateMode::ECONOMICAL}, - {"CONSERVATIVE", FeeEstimateMode::CONSERVATIVE}, - }; - auto mode = fee_modes.find(mode_string); - - if (mode == fee_modes.end()) return false; - - fee_estimate_mode = mode->second; - return true; -} - /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their diff --git a/src/policy/fees.h b/src/policy/fees.h index c8472a12f5..6e61f76178 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -46,8 +46,6 @@ enum class FeeReason { MAXTXFEE, }; -std::string StringForFeeReason(FeeReason reason); - /* Used to determine type of fee estimation requested */ enum class FeeEstimateMode { UNSET, //!< Use default settings based on other criteria @@ -55,8 +53,6 @@ enum class FeeEstimateMode { CONSERVATIVE, //!< Force estimateSmartFee to use conservative estimates }; -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); - /* Used to return detailed information about a feerate bucket */ struct EstimatorBucket { diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index d4cc538492..63a3d06267 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,8 +8,8 @@ #include <policy/policy.h> #include <consensus/validation.h> -#include <validation.h> #include <coins.h> +#include <policy/settings.h> #include <tinyformat.h> #include <util/system.h> #include <util/strencodings.h> @@ -59,7 +59,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) std::vector<std::vector<unsigned char> > vSolutions; whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD || whichType == TX_WITNESS_UNKNOWN) { + if (whichType == TX_NONSTANDARD) { return false; } else if (whichType == TX_MULTISIG) { unsigned char m = vSolutions.front()[0]; @@ -77,7 +77,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) return true; } -bool IsStandardTx(const CTransaction& tx, std::string& reason) +bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason) { if (tx.nVersion > CTransaction::MAX_STANDARD_VERSION || tx.nVersion < 1) { reason = "version"; @@ -123,10 +123,10 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason) if (whichType == TX_NULL_DATA) nDataOut++; - else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { + else if ((whichType == TX_MULTISIG) && (!permit_bare_multisig)) { reason = "bare-multisig"; return false; - } else if (IsDust(txout, ::dustRelayFee)) { + } else if (IsDust(txout, dust_relay_fee)) { reason = "dust"; return false; } @@ -239,21 +239,17 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) return true; } -CFeeRate incrementalRelayFee = CFeeRate(DEFAULT_INCREMENTAL_RELAY_FEE); -CFeeRate dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); -unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; - -int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost) +int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost, unsigned int bytes_per_sigop) { - return (std::max(nWeight, nSigOpCost * nBytesPerSigOp) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; + return (std::max(nWeight, nSigOpCost * bytes_per_sigop) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; } -int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost) +int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost, unsigned int bytes_per_sigop) { - return GetVirtualTransactionSize(GetTransactionWeight(tx), nSigOpCost); + return GetVirtualTransactionSize(GetTransactionWeight(tx), nSigOpCost, bytes_per_sigop); } -int64_t GetVirtualTransactionInputSize(const CTxIn& txin, int64_t nSigOpCost) +int64_t GetVirtualTransactionInputSize(const CTxIn& txin, int64_t nSigOpCost, unsigned int bytes_per_sigop) { - return GetVirtualTransactionSize(GetTransactionInputWeight(txin), nSigOpCost); + return GetVirtualTransactionSize(GetTransactionInputWeight(txin), nSigOpCost, bytes_per_sigop); } diff --git a/src/policy/policy.h b/src/policy/policy.h index 3d47ac1267..ebe040f0ea 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -34,6 +34,8 @@ static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300; static const unsigned int DEFAULT_INCREMENTAL_RELAY_FEE = 1000; /** Default for -bytespersigop */ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; +/** Default for -permitbaremultisig */ +static const bool DEFAULT_PERMIT_BAREMULTISIG = true; /** The maximum number of witness stack items in a standard P2WSH script */ static const unsigned int MAX_STANDARD_P2WSH_STACK_ITEMS = 100; /** The maximum size of each witness stack item in a standard P2WSH script */ @@ -84,7 +86,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); * Check for standard transaction types * @return True if all outputs (scriptPubKeys) use only standard transaction forms */ -bool IsStandardTx(const CTransaction& tx, std::string& reason); +bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason); /** * Check for standard transaction types * @param[in] mapInputs Map of previous transactions that have outputs we're spending @@ -98,13 +100,19 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) */ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs); -extern CFeeRate incrementalRelayFee; -extern CFeeRate dustRelayFee; -extern unsigned int nBytesPerSigOp; - /** Compute the virtual transaction size (weight reinterpreted as bytes). */ -int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost); -int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost = 0); -int64_t GetVirtualTransactionInputSize(const CTxIn& tx, int64_t nSigOpCost = 0); +int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost, unsigned int bytes_per_sigop); +int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost, unsigned int bytes_per_sigop); +int64_t GetVirtualTransactionInputSize(const CTxIn& tx, int64_t nSigOpCost, unsigned int bytes_per_sigop); + +static inline int64_t GetVirtualTransactionSize(const CTransaction& tx) +{ + return GetVirtualTransactionSize(tx, 0, 0); +} + +static inline int64_t GetVirtualTransactionInputSize(const CTxIn& tx) +{ + return GetVirtualTransactionInputSize(tx, 0, 0); +} #endif // BITCOIN_POLICY_POLICY_H diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 0dc130d104..b4b8341d77 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -3,18 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/rbf.h> +#include <util/rbf.h> -bool SignalsOptInRBF(const CTransaction &tx) -{ - for (const CTxIn &txin : tx.vin) { - if (txin.nSequence <= MAX_BIP125_RBF_SEQUENCE) { - return true; - } - } - return false; -} - -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 581f489e12..0707b0044f 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -7,22 +7,16 @@ #include <txmempool.h> -static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; - enum class RBFTransactionState { UNKNOWN, REPLACEABLE_BIP125, FINAL }; -// Check whether the sequence numbers on this transaction are signaling -// opt-in to replace-by-fee, according to BIP 125 -bool SignalsOptInRBF(const CTransaction &tx); - // Determine whether an in-mempool transaction is signaling opt-in to RBF // according to BIP 125 // This involves checking sequence numbers of the transaction, as well // as the sequence numbers of all in-mempool ancestors. -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/policy/settings.cpp b/src/policy/settings.cpp new file mode 100644 index 0000000000..e8e1559407 --- /dev/null +++ b/src/policy/settings.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/settings.h> + +#include <policy/feerate.h> +#include <policy/policy.h> + +bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; +CFeeRate incrementalRelayFee = CFeeRate(DEFAULT_INCREMENTAL_RELAY_FEE); +CFeeRate dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); +unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; diff --git a/src/policy/settings.h b/src/policy/settings.h new file mode 100644 index 0000000000..30a7189c93 --- /dev/null +++ b/src/policy/settings.h @@ -0,0 +1,35 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_SETTINGS_H +#define BITCOIN_POLICY_SETTINGS_H + +#include <policy/policy.h> + +class CFeeRate; +class CTransaction; + +// Policy settings which are configurable at runtime. +extern CFeeRate incrementalRelayFee; +extern CFeeRate dustRelayFee; +extern unsigned int nBytesPerSigOp; +extern bool fIsBareMultisigStd; + +static inline bool IsStandardTx(const CTransaction& tx, std::string& reason) +{ + return IsStandardTx(tx, ::fIsBareMultisigStd, ::dustRelayFee, reason); +} + +static inline int64_t GetVirtualTransactionSize(int64_t weight, int64_t sigop_cost) +{ + return GetVirtualTransactionSize(weight, sigop_cost, ::nBytesPerSigOp); +} + +static inline int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t sigop_cost) +{ + return GetVirtualTransactionSize(tx, sigop_cost, ::nBytesPerSigOp); +} + +#endif // BITCOIN_POLICY_SETTINGS_H diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index f6f8e31363..aad991e2f1 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -222,6 +222,10 @@ inline void UnserializeTransaction(TxType& tx, Stream& s) { for (size_t i = 0; i < tx.vin.size(); i++) { s >> tx.vin[i].scriptWitness.stack; } + if (!tx.HasWitness()) { + /* It's illegal to encode witnesses when all witness stacks are empty. */ + throw std::ios_base::failure("Superfluous witness record"); + } } if (flags) { /* Unknown flag in the serialization */ diff --git a/src/psbt.cpp b/src/psbt.cpp index 0fb7d49d7d..97bda51a63 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -2,9 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> +#include <consensus/tx_verify.h> +#include <policy/policy.h> #include <psbt.h> #include <util/strencodings.h> +#include <numeric> + PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) { inputs.resize(tx.vin.size()); @@ -205,7 +210,7 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; } -bool PSBTInputSigned(PSBTInput& input) +bool PSBTInputSigned(const PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } @@ -325,3 +330,41 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector return TransactionError::OK; } + +std::string PSBTRoleName(PSBTRole role) { + switch (role) { + case PSBTRole::UPDATER: return "updater"; + case PSBTRole::SIGNER: return "signer"; + case PSBTRole::FINALIZER: return "finalizer"; + case PSBTRole::EXTRACTOR: return "extractor"; + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) +{ + bool invalid; + std::string tx_data = DecodeBase64(base64_tx, &invalid); + if (invalid) { + error = "invalid base64"; + return false; + } + return DecodeRawPSBT(psbt, tx_data, error); +} + +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +{ + CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + try { + ss_data >> psbt; + if (!ss_data.empty()) { + error = "extra data after PSBT"; + return false; + } + } catch (const std::exception& e) { + error = e.what(); + return false; + } + return true; +} diff --git a/src/psbt.h b/src/psbt.h index c889dad361..1bc1e91a84 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -7,6 +7,8 @@ #include <attributes.h> #include <node/transaction.h> +#include <optional.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <pubkey.h> #include <script/sign.h> @@ -548,8 +550,17 @@ struct PartiallySignedTransaction } }; +enum class PSBTRole { + UPDATER, + SIGNER, + FINALIZER, + EXTRACTOR +}; + +std::string PSBTRoleName(PSBTRole role); + /** Checks whether a PSBTInput is already signed. */ -bool PSBTInputSigned(PSBTInput& input); +bool PSBTInputSigned(const PSBTInput& input); /** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); @@ -580,4 +591,9 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti */ NODISCARD TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); +//! Decode a base64ed PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); +//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); + #endif // BITCOIN_PSBT_H diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 726dafccb0..50d6afabcd 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -107,7 +107,7 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, ui->newAddress->setVisible(true); break; case ReceivingTab: - ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. It is recommended to use a new receiving address for each transaction.")); + ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. Use the 'Create new receiving address' button in the receive tab to create new addresses.")); ui->deleteAddress->setVisible(false); ui->newAddress->setVisible(false); break; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 1b063771ef..81255aaae9 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,6 +30,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <noui.h> +#include <util/threadnames.h> #include <rpc/server.h> #include <ui_interface.h> #include <uint256.h> @@ -149,6 +150,7 @@ void BitcoinCore::initialize() try { qDebug() << __func__ << ": Running initialization in thread"; + util::ThreadRename("qt-init"); bool rv = m_node.appInitMain(); Q_EMIT initializeResult(rv); } catch (const std::exception& e) { @@ -423,6 +425,7 @@ int GuiMain(int argc, char* argv[]) std::tie(argc, argv) = winArgs.get(); #endif SetupEnvironment(); + util::ThreadRename("main"); std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 27b4c182f9..88a35534c2 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -11,7 +11,6 @@ #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <clientversion.h> #include <interfaces/handler.h> #include <interfaces/node.h> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 2f916d0b44..0d280f2993 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -108,7 +108,7 @@ </size> </property> <property name="text"> - <string>&Request payment</string> + <string>&Create new receiving address</string> </property> <property name="icon"> <iconset resource="../bitcoin.qrc"> @@ -189,7 +189,7 @@ </widget> </item> <item> - <widget class="QCheckBox" name="useBech32"> + <widget class="QCheckBox" name="useLegacyAddress"> <property name="sizePolicy"> <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> <horstretch>0</horstretch> @@ -206,10 +206,10 @@ <enum>Qt::StrongFocus</enum> </property> <property name="toolTip"> - <string>Native segwit addresses (aka Bech32 or BIP-173) reduce your transaction fees later on and offer better protection against typos, but old wallets don't support them. When unchecked, an address compatible with older wallets will be created instead.</string> + <string>Native segwit addresses (aka Bech32 or BIP-173) reduce your transaction fees later on and offer better protection against typos, but old wallets don't support them. When checked, an address compatible with older wallets will be created instead.</string> </property> <property name="text"> - <string>Generate native segwit (Bech32) address</string> + <string>Generate legacy address</string> </property> </widget> </item> @@ -360,7 +360,7 @@ <tabstops> <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> - <tabstop>useBech32</tabstop> + <tabstop>useLegacyAddress</tabstop> <tabstop>reqMessage</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index ba0a5abdf3..45f21d50fc 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -175,7 +175,9 @@ bool parseBitcoinURI(QString uri, SendCoinsRecipient *out) QString formatBitcoinURI(const SendCoinsRecipient &info) { - QString ret = QString("bitcoin:%1").arg(info.address); + bool bech_32 = info.address.startsWith(QString::fromStdString(Params().Bech32HRP() + "1")); + + QString ret = QString("bitcoin:%1").arg(bech_32 ? info.address.toUpper() : info.address); int paramCount = 0; if (info.amount) @@ -244,6 +246,11 @@ QList<QModelIndex> getEntryData(QAbstractItemView *view, int column) return view->selectionModel()->selectedRows(column); } +QString getDefaultDataDirectory() +{ + return boostPathToQString(GetDefaultDataDir()); +} + QString getSaveFileName(QWidget *parent, const QString &caption, const QString &dir, const QString &filter, QString *selectedSuffixOut) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index cbec73a882..bea4a83494 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -79,6 +79,11 @@ namespace GUIUtil void setClipboard(const QString& str); + /** + * Determine default data directory for operating system. + */ + QString getDefaultDataDirectory(); + /** Get save filename, mimics QFileDialog::getSaveFileName, except that it appends a default suffix when no suffix is provided by the user. diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 499af9fa07..c595361934 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -168,7 +168,7 @@ QString Intro::getDataDirectory() void Intro::setDataDirectory(const QString &dataDir) { ui->dataDirectory->setText(dataDir); - if(dataDir == getDefaultDataDirectory()) + if(dataDir == GUIUtil::getDefaultDataDirectory()) { ui->dataDirDefault->setChecked(true); ui->dataDirectory->setEnabled(false); @@ -180,11 +180,6 @@ void Intro::setDataDirectory(const QString &dataDir) } } -QString Intro::getDefaultDataDirectory() -{ - return GUIUtil::boostPathToQString(GetDefaultDataDir()); -} - bool Intro::pickDataDirectory(interfaces::Node& node) { QSettings settings; @@ -193,7 +188,7 @@ bool Intro::pickDataDirectory(interfaces::Node& node) if(!gArgs.GetArg("-datadir", "").empty()) return true; /* 1) Default data directory for operating system */ - QString dataDir = getDefaultDataDirectory(); + QString dataDir = GUIUtil::getDefaultDataDirectory(); /* 2) Allow QSettings to override default dir */ dataDir = settings.value("strDataDir", dataDir).toString(); @@ -239,7 +234,7 @@ bool Intro::pickDataDirectory(interfaces::Node& node) * override -datadir in the bitcoin.conf file in the default data directory * (to be consistent with bitcoind behavior) */ - if(dataDir != getDefaultDataDirectory()) { + if(dataDir != GUIUtil::getDefaultDataDirectory()) { node.softSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting } return true; @@ -293,7 +288,7 @@ void Intro::on_ellipsisButton_clicked() void Intro::on_dataDirDefault_clicked() { - setDataDirectory(getDefaultDataDirectory()); + setDataDirectory(GUIUtil::getDefaultDataDirectory()); } void Intro::on_dataDirCustom_clicked() diff --git a/src/qt/intro.h b/src/qt/intro.h index b537c94f7d..c3b26808d4 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -48,11 +48,6 @@ public: */ static bool pickDataDirectory(interfaces::Node& node); - /** - * Determine default data directory for operating system. - */ - static QString getDefaultDataDirectory(); - Q_SIGNALS: void requestCheck(); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 9094aeff56..40dc7bf400 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -154,6 +154,10 @@ void OptionsDialog::setModel(OptionsModel *_model) if (_model->isRestartRequired()) showRestartWarning(true); + // Prune values are in GB to be consistent with intro.cpp + static constexpr uint64_t nMinDiskSpace = (MIN_DISK_SPACE_FOR_BLOCK_FILES / GB_BYTES) + (MIN_DISK_SPACE_FOR_BLOCK_FILES % GB_BYTES) ? 1 : 0; + ui->pruneSize->setRange(nMinDiskSpace, std::numeric_limits<int>::max()); + QString strLabel = _model->getOverriddenByCommandLine(); if (strLabel.isEmpty()) strLabel = tr("none"); @@ -164,10 +168,6 @@ void OptionsDialog::setModel(OptionsModel *_model) mapper->toFirst(); updateDefaultProxyNets(); - - // Prune values are in GB to be consistent with intro.cpp - static constexpr uint64_t nMinDiskSpace = (MIN_DISK_SPACE_FOR_BLOCK_FILES / GB_BYTES) + (MIN_DISK_SPACE_FOR_BLOCK_FILES % GB_BYTES) ? 1 : 0; - ui->pruneSize->setRange(nMinDiskSpace, _model->node().getAssumedBlockchainSize()); } /* warn when one of the following settings changes by user action (placed here so init via mapper doesn't trigger them) */ diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 62496a57f4..5b4fb4cc18 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -17,7 +17,6 @@ #include <net.h> #include <netbase.h> #include <txdb.h> // for -dbcache defaults -#include <qt/intro.h> #include <QNetworkProxy> #include <QSettings> @@ -110,7 +109,7 @@ void OptionsModel::Init(bool resetSettings) addOverriddenOption("-par"); if (!settings.contains("strDataDir")) - settings.setValue("strDataDir", Intro::getDefaultDataDirectory()); + settings.setValue("strDataDir", GUIUtil::getDefaultDataDirectory()); // Wallet #ifdef ENABLE_WALLET @@ -187,7 +186,7 @@ void OptionsModel::Reset() BackupSettings(GetDataDir(true) / "guisettings.ini.bak", settings); // Save the strDataDir setting - QString dataDir = Intro::getDefaultDataDirectory(); + QString dataDir = GUIUtil::getDefaultDataDirectory(); dataDir = settings.value("strDataDir", dataDir).toString(); // Remove all entries from our QSettings object diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 22a79a12bb..fc58090dcd 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -95,9 +95,9 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH, this); if (model->wallet().getDefaultAddressType() == OutputType::BECH32) { - ui->useBech32->setCheckState(Qt::Checked); + ui->useLegacyAddress->setCheckState(Qt::Unchecked); } else { - ui->useBech32->setCheckState(Qt::Unchecked); + ui->useLegacyAddress->setCheckState(Qt::Checked); } // Set the button to be enabled or disabled based on whether the wallet can give out new addresses. @@ -150,7 +150,7 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString label = ui->reqLabel->text(); /* Generate new receiving address */ OutputType address_type; - if (ui->useBech32->isChecked()) { + if (!ui->useLegacyAddress->isChecked()) { address_type = OutputType::BECH32; } else { address_type = model->wallet().getDefaultAddressType(); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index fc1e14b031..8b6dcf0445 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -21,8 +21,6 @@ #include <util/strencodings.h> #include <util/system.h> -#include <openssl/crypto.h> - #include <univalue.h> #ifdef ENABLE_WALLET @@ -69,7 +67,6 @@ const QStringList historyFilter = QStringList() << "importmulti" << "sethdseed" << "signmessagewithprivkey" - << "signrawtransaction" << "signrawtransactionwithkey" << "walletpassphrase" << "walletpassphrasechange" diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 6e00ab755c..8a0b265834 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -578,7 +578,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::AbsurdFee: - msgParams.first = tr("A fee higher than %1 is considered an absurdly high fee.").arg(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), model->node().getMaxTxFee())); + msgParams.first = tr("A fee higher than %1 is considered an absurdly high fee.").arg(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), model->wallet().getDefaultMaxTxFee())); break; case WalletModel::PaymentRequestExpired: msgParams.first = tr("Payment request expired."); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index d37a78fa8c..64cc85d623 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -11,7 +11,7 @@ #include <qt/walletmodel.h> #include <key_io.h> -#include <validation.h> // For strMessageMagic +#include <util/validation.h> // For strMessageMagic #include <wallet/wallet.h> #include <string> diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 7f5e92ea9f..2ba1c2604c 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,6 +1,6 @@ #include <qt/test/addressbooktests.h> #include <qt/test/util.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <interfaces/chain.h> #include <interfaces/node.h> @@ -58,7 +58,7 @@ void TestAddAddressesToSendBook() { TestChain100Setup test; auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index f8a9c25303..b0bd89b290 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -12,7 +12,7 @@ #include <rpc/register.h> #include <rpc/server.h> #include <qt/rpcconsole.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <univalue.h> #include <util/system.h> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 2b50a2ba81..9e3518fd53 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -15,7 +15,7 @@ #include <qt/transactionview.h> #include <qt/walletmodel.h> #include <key_io.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <validation.h> #include <wallet/wallet.h> #include <qt/overviewpage.h> @@ -135,7 +135,7 @@ void TestGUI() test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); { diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index fab86a7912..019bd65823 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -9,9 +9,11 @@ #include <algorithm> +#include <QApplication> #include <QMessageBox> #include <QMutexLocker> #include <QThread> +#include <QWindow> WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) : QObject(parent) @@ -97,7 +99,17 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal m_wallets.push_back(wallet_model); connect(wallet_model, &WalletModel::unload, [this, wallet_model] { - removeAndDeleteWallet(wallet_model); + // Defer removeAndDeleteWallet when no modal widget is active. + // TODO: remove this workaround by removing usage of QDiallog::exec. + if (QApplication::activeModalWidget()) { + connect(qApp, &QApplication::focusWindowChanged, wallet_model, [this, wallet_model]() { + if (!QApplication::activeModalWidget()) { + removeAndDeleteWallet(wallet_model); + } + }, Qt::QueuedConnection); + } else { + removeAndDeleteWallet(wallet_model); + } }); // Re-emit coinsSent signal from wallet model. diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 466f2278eb..94413547d4 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -40,15 +40,11 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) this->clientModel = _clientModel; } -bool WalletFrame::addWallet(WalletModel *walletModel) +void WalletFrame::addWallet(WalletModel *walletModel) { - if (!gui || !clientModel || !walletModel) { - return false; - } + if (!gui || !clientModel || !walletModel) return; - if (mapWalletViews.count(walletModel) > 0) { - return false; - } + if (mapWalletViews.count(walletModel) > 0) return; WalletView *walletView = new WalletView(platformStyle, this); walletView->setBitcoinGUI(gui); @@ -72,31 +68,25 @@ bool WalletFrame::addWallet(WalletModel *walletModel) }); connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); - - return true; } -bool WalletFrame::setCurrentWallet(WalletModel* wallet_model) +void WalletFrame::setCurrentWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.value(wallet_model); walletStack->setCurrentWidget(walletView); assert(walletView); walletView->updateEncryptionStatus(); - return true; } -bool WalletFrame::removeWallet(WalletModel* wallet_model) +void WalletFrame::removeWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.take(wallet_model); walletStack->removeWidget(walletView); delete walletView; - return true; } void WalletFrame::removeAllWallets() diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 6a74fde9fd..156653f47d 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -36,9 +36,9 @@ public: void setClientModel(ClientModel *clientModel); - bool addWallet(WalletModel *walletModel); - bool setCurrentWallet(WalletModel* wallet_model); - bool removeWallet(WalletModel* wallet_model); + void addWallet(WalletModel *walletModel); + void setCurrentWallet(WalletModel* wallet_model); + void removeWallet(WalletModel* wallet_model); void removeAllWallets(); bool handlePaymentRequest(const SendCoinsRecipient& recipient); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index f4f3be8f43..fd392b7cf7 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -222,9 +222,9 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact } // reject absurdly high fee. (This can never happen because the - // wallet caps the fee at maxTxFee. This merely serves as a + // wallet caps the fee at m_default_max_tx_fee. This merely serves as a // belt-and-suspenders check) - if (nFeeRequired > m_node.getMaxTxFee()) + if (nFeeRequired > m_wallet->getDefaultMaxTxFee()) return AbsurdFee; } diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eb3b0baf08..2694d67800 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -29,7 +29,7 @@ std::unique_ptr<interfaces::PendingWalletTx>& WalletModelTransaction::getWtx() unsigned int WalletModelTransaction::getTransactionSize() { - return wtx ? wtx->getVirtualSize() : 0; + return wtx ? GetVirtualTransactionSize(wtx->get()) : 0; } CAmount WalletModelTransaction::getTransactionFee() const diff --git a/src/rest.cpp b/src/rest.cpp index 326f7ae1d2..baad3b2ce9 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -300,7 +300,7 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) switch (rf) { case RetFormat::JSON: { - UniValue mempoolInfoObject = mempoolInfoToJSON(); + UniValue mempoolInfoObject = MempoolInfoToJSON(::mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -322,7 +322,7 @@ static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPar switch (rf) { case RetFormat::JSON: { - UniValue mempoolObject = mempoolToJSON(true); + UniValue mempoolObject = MempoolToJSON(::mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7e8e5e07d0..f140999622 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -7,13 +7,14 @@ #include <amount.h> #include <base58.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <coins.h> #include <consensus/validation.h> #include <core_io.h> #include <hash.h> +#include <index/blockfilterindex.h> #include <index/txindex.h> #include <key_io.h> #include <policy/feerate.h> @@ -29,6 +30,7 @@ #include <txmempool.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/validation.h> #include <validation.h> #include <validationinterface.h> #include <versionbitsinfo.h> @@ -91,6 +93,9 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) { + // Serialize passed information without accessing chain state of the active chain! + AssertLockNotHeld(cs_main); // For performance reasons + UniValue result(UniValue::VOBJ); result.pushKV("hash", blockindex->GetBlockHash().GetHex()); const CBlockIndex* pnext; @@ -117,6 +122,9 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) { + // Serialize passed information without accessing chain state of the active chain! + AssertLockNotHeld(cs_main); // For performance reasons + UniValue result(UniValue::VOBJ); result.pushKV("hash", blockindex->GetBlockHash().GetHex()); const CBlockIndex* pnext; @@ -378,7 +386,9 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::string EntryDescriptionString() { - return " \"size\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + return " \"vsize\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + " \"size\" : n, (numeric) (DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" + " size will be completely removed in v0.20.\n" " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)\n" " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority (DEPRECATED)\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" @@ -405,9 +415,9 @@ static std::string EntryDescriptionString() " \"bip125-replaceable\" : true|false, (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee)\n"; } -static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCKS_REQUIRED(::mempool.cs) +static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { - AssertLockHeld(mempool.cs); + AssertLockHeld(pool.cs); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -416,7 +426,8 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); info.pushKV("fees", fees); - info.pushKV("size", (int)e.GetTxSize()); + info.pushKV("vsize", (int)e.GetTxSize()); + if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); info.pushKV("time", e.GetTime()); @@ -427,12 +438,12 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); - info.pushKV("wtxid", mempool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); const CTransaction& tx = e.GetTx(); std::set<std::string> setDepends; for (const CTxIn& txin : tx.vin) { - if (mempool.exists(txin.prevout.hash)) + if (pool.exists(txin.prevout.hash)) setDepends.insert(txin.prevout.hash.ToString()); } @@ -445,8 +456,8 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter &it = mempool.mapTx.find(tx.GetHash()); - const CTxMemPool::setEntries &setChildren = mempool.GetMemPoolChildren(it); + const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); + const CTxMemPool::setEntries& setChildren = pool.GetMemPoolChildren(it); for (CTxMemPool::txiter childiter : setChildren) { spent.push_back(childiter->GetTx().GetHash().ToString()); } @@ -455,7 +466,7 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK // Add opt-in RBF status bool rbfStatus = false; - RBFTransactionState rbfState = IsRBFOptIn(tx, mempool); + RBFTransactionState rbfState = IsRBFOptIn(tx, pool); if (rbfState == RBFTransactionState::UNKNOWN) { throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { @@ -465,25 +476,21 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("bip125-replaceable", rbfStatus); } -UniValue mempoolToJSON(bool fVerbose) +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose) { - if (fVerbose) - { - LOCK(mempool.cs); + if (verbose) { + LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : mempool.mapTx) - { + for (const CTxMemPoolEntry& e : pool.mapTx) { const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(pool, info, e); o.pushKV(hash.ToString(), info); } return o; - } - else - { + } else { std::vector<uint256> vtxid; - mempool.queryHashes(vtxid); + pool.queryHashes(vtxid); UniValue a(UniValue::VARR); for (const uint256& hash : vtxid) @@ -525,7 +532,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return mempoolToJSON(fVerbose); + return MempoolToJSON(::mempool, fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -591,7 +598,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *ancestorIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -661,7 +668,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *descendantIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -700,7 +707,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); return info; } @@ -823,9 +830,7 @@ static CBlock GetBlockChecked(const CBlockIndex* pblockindex) static UniValue getblock(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( - RPCHelpMan{"getblock", + const RPCHelpMan help{"getblock", "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block <hash>.\n" "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", @@ -877,9 +882,11 @@ static UniValue getblock(const JSONRPCRequest& request) HelpExampleCli("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") }, - }.ToString()); + }; - LOCK(cs_main); + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } uint256 hash(ParseHashV(request.params[0], "blockhash")); @@ -891,12 +898,20 @@ static UniValue getblock(const JSONRPCRequest& request) verbosity = request.params[1].get_bool() ? 1 : 0; } - const CBlockIndex* pblockindex = LookupBlockIndex(hash); - if (!pblockindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } + CBlock block; + const CBlockIndex* pblockindex; + const CBlockIndex* tip; + { + LOCK(cs_main); + pblockindex = LookupBlockIndex(hash); + tip = chainActive.Tip(); + + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } - const CBlock block = GetBlockChecked(pblockindex); + block = GetBlockChecked(pblockindex); + } if (verbosity <= 0) { @@ -906,7 +921,7 @@ static UniValue getblock(const JSONRPCRequest& request) return strHex; } - return blockToJSON(block, chainActive.Tip(), pblockindex, verbosity >= 2); + return blockToJSON(block, tip, pblockindex, verbosity >= 2); } struct CCoinsStats @@ -1011,7 +1026,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -1485,15 +1500,18 @@ static UniValue getchaintips(const JSONRPCRequest& request) return res; } -UniValue mempoolInfoToJSON() +UniValue MempoolInfoToJSON(const CTxMemPool& pool) { + // Make sure this call is atomic in the pool. + LOCK(pool.cs); UniValue ret(UniValue::VOBJ); - ret.pushKV("size", (int64_t) mempool.size()); - ret.pushKV("bytes", (int64_t) mempool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t) mempool.DynamicMemoryUsage()); + ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(std::max(mempool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); + ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); return ret; @@ -1508,6 +1526,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) {}, RPCResult{ "{\n" + " \"loaded\": true|false (boolean) True if the mempool is fully loaded\n" " \"size\": xxxxx, (numeric) Current tx count\n" " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" @@ -1522,7 +1541,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.ToString()); - return mempoolInfoToJSON(); + return MempoolInfoToJSON(::mempool); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1583,15 +1602,15 @@ static UniValue invalidateblock(const JSONRPCRequest& request) uint256 hash(ParseHashV(request.params[0], "blockhash")); CValidationState state; + CBlockIndex* pblockindex; { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + pblockindex = LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - - InvalidateBlock(state, Params(), pblockindex); } + InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { ActivateBestChain(state, Params()); @@ -2058,11 +2077,11 @@ static UniValue savemempool(const JSONRPCRequest& request) }.ToString()); } - if (!g_is_mempool_loaded) { + if (!::mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool()) { + if (!DumpMempool(::mempool)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } @@ -2297,6 +2316,85 @@ UniValue scantxoutset(const JSONRPCRequest& request) return result; } +static UniValue getblockfilter(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + throw std::runtime_error( + RPCHelpMan{"getblockfilter", + "\nRetrieve a BIP 157 content filter for a particular block.\n", + { + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hash of the block"}, + {"filtertype", RPCArg::Type::STR, /*default*/ "basic", "The type name of the filter"}, + }, + RPCResult{ + "{\n" + " \"filter\" : (string) the hex-encoded filter data\n" + " \"header\" : (string) the hex-encoded filter header\n" + "}\n" + }, + RPCExamples{ + HelpExampleCli("getblockfilter", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" \"basic\"") + } + }.ToString() + ); + } + + uint256 block_hash = ParseHashV(request.params[0], "blockhash"); + std::string filtertype_name = "basic"; + if (!request.params[1].isNull()) { + filtertype_name = request.params[1].get_str(); + } + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(filtertype_name, filtertype)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); + } + + const CBlockIndex* block_index; + bool block_was_connected; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block_hash); + if (!block_index) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS); + } + + bool index_ready = index->BlockUntilSyncedToCurrentChain(); + + BlockFilter filter; + uint256 filter_header; + if (!index->LookupFilter(block_index, filter) || + !index->LookupFilterHeader(block_index, filter_header)) { + int err_code; + std::string errmsg = "Filter not found."; + + if (!block_was_connected) { + err_code = RPC_INVALID_ADDRESS_OR_KEY; + errmsg += " Block was not connected to active chain."; + } else if (!index_ready) { + err_code = RPC_MISC_ERROR; + errmsg += " Block filters are still in the process of being indexed."; + } else { + err_code = RPC_INTERNAL_ERROR; + errmsg += " This error is unexpected and indicates index corruption."; + } + + throw JSONRPCError(err_code, errmsg); + } + + UniValue ret(UniValue::VOBJ); + ret.pushKV("filter", HexStr(filter.GetEncodedFilter())); + ret.pushKV("header", filter_header.GetHex()); + return ret; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2324,6 +2422,7 @@ static const CRPCCommand commands[] = { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, + { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} }, /* Not shown in help */ { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 529132d033..ff461fbcbc 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -5,12 +5,17 @@ #ifndef BITCOIN_RPC_BLOCKCHAIN_H #define BITCOIN_RPC_BLOCKCHAIN_H -#include <vector> -#include <stdint.h> #include <amount.h> +#include <sync.h> + +#include <stdint.h> +#include <vector> + +extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; +class CTxMemPool; class UniValue; static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -27,16 +32,16 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false); +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); /** Mempool information to JSON */ -UniValue mempoolInfoToJSON(); +UniValue MempoolInfoToJSON(const CTxMemPool& pool); /** Mempool to JSON */ -UniValue mempoolToJSON(bool fVerbose = false); +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false); /** Block header to JSON */ -UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex); +UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) LOCKS_EXCLUDED(cs_main); /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 82424ab521..4144a17bc3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -28,8 +28,6 @@ public: static const CRPCConvertParam vRPCConvertParams[] = { { "setmocktime", 0, "timestamp" }, - { "generate", 0, "nblocks" }, - { "generate", 1, "maxtries" }, { "generatetoaddress", 0, "nblocks" }, { "generatetoaddress", 2, "maxtries" }, { "getnetworkhashps", 0, "nblocks" }, @@ -94,8 +92,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, { "sendrawtransaction", 1, "allowhighfees" }, + { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "allowhighfees" }, + { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 2, "iswitness" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 6625a03bbd..4de738a756 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -16,13 +16,16 @@ #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> -#include <rpc/mining.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/script.h> #include <shutdown.h> #include <txmempool.h> +#include <univalue.h> +#include <util/fees.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/validation.h> #include <validation.h> #include <validationinterface.h> #include <versionbitsinfo.h> @@ -98,7 +101,7 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } -UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) +static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { static const int nInnerLoopCount = 0x10000; int nHeightEnd = 0; @@ -113,7 +116,7 @@ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGen UniValue blockHashes(UniValue::VARR); while (nHeight < nHeightEnd && !ShutdownRequested()) { - std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(Params()).CreateNewBlock(coinbaseScript->reserveScript)); + std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; @@ -136,12 +139,6 @@ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGen throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); ++nHeight; blockHashes.push_back(pblock->GetHash().GetHex()); - - //mark script as important because it was used at least for one coinbase output if the script came from the wallet - if (keepScript) - { - coinbaseScript->KeepScript(); - } } return blockHashes; } @@ -179,10 +176,9 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); } - std::shared_ptr<CReserveScript> coinbaseScript = std::make_shared<CReserveScript>(); - coinbaseScript->reserveScript = GetScriptForDestination(destination); + CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(coinbaseScript, nGenerate, nMaxTries, false); + return generateBlocks(coinbase_script, nGenerate, nMaxTries); } static UniValue getmininginfo(const JSONRPCRequest& request) @@ -444,10 +440,10 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Bitcoin is not connected!"); + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Bitcoin is downloading blocks..."); + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; @@ -843,7 +839,8 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); bool conservative = true; if (!request.params[1].isNull()) { FeeEstimateMode fee_mode; @@ -915,7 +912,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); double threshold = 0.95; if (!request.params[1].isNull()) { threshold = request.params[1].get_real(); diff --git a/src/rpc/mining.h b/src/rpc/mining.h deleted file mode 100644 index be9a973315..0000000000 --- a/src/rpc/mining.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2017 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_RPC_MINING_H -#define BITCOIN_RPC_MINING_H - -#include <script/script.h> - -#include <univalue.h> - -/** Generate blocks (mine) */ -UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript); - -#endif diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 822a0beef9..bfb559f0db 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -20,6 +20,7 @@ #include <timedata.h> #include <util/system.h> #include <util/strencodings.h> +#include <util/validation.h> #include <warnings.h> #include <stdint.h> @@ -34,11 +35,7 @@ static UniValue validateaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( RPCHelpMan{"validateaddress", - "\nReturn information about the given bitcoin address.\n" - "DEPRECATION WARNING: Parts of this command have been deprecated and moved to getaddressinfo. Clients must\n" - "transition to using getaddressinfo to access this information before upgrading to v0.18. The following deprecated\n" - "fields have moved to getaddressinfo and will only be shown here with -deprecatedrpc=validateaddress: ismine, iswatchonly,\n" - "script, hex, pubkeys, sigsrequired, pubkey, addresses, embedded, iscompressed, account, timestamp, hdkeypath, kdmasterkeyid.\n", + "\nReturn information about the given bitcoin address.\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, }, @@ -232,8 +229,8 @@ UniValue deriveaddresses(const JSONRPCRequest& request) range_end = range.second; } - FlatSigningProvider provider; - auto desc = Parse(desc_str, provider, /* require_checksum = */ true); + FlatSigningProvider key_provider; + auto desc = Parse(desc_str, key_provider, /* require_checksum = */ true); if (!desc) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); } @@ -249,8 +246,9 @@ UniValue deriveaddresses(const JSONRPCRequest& request) UniValue addresses(UniValue::VARR); for (int i = range_begin; i <= range_end; ++i) { + FlatSigningProvider provider; std::vector<CScript> scripts; - if (!desc->Expand(i, provider, scripts, provider)) { + if (!desc->Expand(i, key_provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index c7b3478f44..e8cdce623c 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -12,6 +12,7 @@ #include <net_processing.h> #include <netbase.h> #include <policy/policy.h> +#include <policy/settings.h> #include <rpc/protocol.h> #include <rpc/util.h> #include <sync.h> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index d19afaa8a1..78d7bbc80c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,15 +11,19 @@ #include <core_io.h> #include <index/txindex.h> #include <init.h> +#include <interfaces/chain.h> #include <key_io.h> #include <keystore.h> #include <merkleblock.h> +#include <node/coin.h> +#include <node/psbt.h> #include <node/transaction.h> #include <policy/policy.h> #include <policy/rbf.h> +#include <policy/settings.h> #include <primitives/transaction.h> #include <psbt.h> -#include <rpc/rawtransaction.h> +#include <rpc/rawtransaction_util.h> #include <rpc/server.h> #include <rpc/util.h> #include <script/script.h> @@ -28,6 +32,7 @@ #include <script/standard.h> #include <uint256.h> #include <util/bip32.h> +#include <util/moneystr.h> #include <util/strencodings.h> #include <validation.h> #include <validationinterface.h> @@ -38,6 +43,11 @@ #include <univalue.h> +/** High fee for sendrawtransaction and testmempoolaccept. + * By default, transaction with a fee higher than this will be rejected by the + * RPCs. This can be overriden with the maxfeerate argument. + */ +constexpr static CAmount DEFAULT_MAX_RAW_TX_FEE{COIN / 10}; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { @@ -352,119 +362,6 @@ static UniValue verifytxoutproof(const JSONRPCRequest& request) return res; } -CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, const UniValue& rbf) -{ - if (inputs_in.isNull() || outputs_in.isNull()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); - - UniValue inputs = inputs_in.get_array(); - const bool outputs_is_obj = outputs_in.isObject(); - UniValue outputs = outputs_is_obj ? outputs_in.get_obj() : outputs_in.get_array(); - - CMutableTransaction rawTx; - - if (!locktime.isNull()) { - int64_t nLockTime = locktime.get_int64(); - if (nLockTime < 0 || nLockTime > LOCKTIME_MAX) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, locktime out of range"); - rawTx.nLockTime = nLockTime; - } - - bool rbfOptIn = rbf.isTrue(); - - for (unsigned int idx = 0; idx < inputs.size(); idx++) { - const UniValue& input = inputs[idx]; - const UniValue& o = input.get_obj(); - - uint256 txid = ParseHashO(o, "txid"); - - const UniValue& vout_v = find_value(o, "vout"); - if (!vout_v.isNum()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); - int nOutput = vout_v.get_int(); - if (nOutput < 0) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); - - uint32_t nSequence; - if (rbfOptIn) { - nSequence = MAX_BIP125_RBF_SEQUENCE; /* CTxIn::SEQUENCE_FINAL - 2 */ - } else if (rawTx.nLockTime) { - nSequence = CTxIn::SEQUENCE_FINAL - 1; - } else { - nSequence = CTxIn::SEQUENCE_FINAL; - } - - // set the sequence number if passed in the parameters object - const UniValue& sequenceObj = find_value(o, "sequence"); - if (sequenceObj.isNum()) { - int64_t seqNr64 = sequenceObj.get_int64(); - if (seqNr64 < 0 || seqNr64 > CTxIn::SEQUENCE_FINAL) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, sequence number is out of range"); - } else { - nSequence = (uint32_t)seqNr64; - } - } - - CTxIn in(COutPoint(txid, nOutput), CScript(), nSequence); - - rawTx.vin.push_back(in); - } - - if (!outputs_is_obj) { - // Translate array of key-value pairs into dict - UniValue outputs_dict = UniValue(UniValue::VOBJ); - for (size_t i = 0; i < outputs.size(); ++i) { - const UniValue& output = outputs[i]; - if (!output.isObject()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, key-value pair not an object as expected"); - } - if (output.size() != 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, key-value pair must contain exactly one key"); - } - outputs_dict.pushKVs(output); - } - outputs = std::move(outputs_dict); - } - - // Duplicate checking - std::set<CTxDestination> destinations; - bool has_data{false}; - - for (const std::string& name_ : outputs.getKeys()) { - if (name_ == "data") { - if (has_data) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, duplicate key: data"); - } - has_data = true; - std::vector<unsigned char> data = ParseHexV(outputs[name_].getValStr(), "Data"); - - CTxOut out(0, CScript() << OP_RETURN << data); - rawTx.vout.push_back(out); - } else { - CTxDestination destination = DecodeDestination(name_); - if (!IsValidDestination(destination)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + name_); - } - - if (!destinations.insert(destination).second) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + name_); - } - - CScript scriptPubKey = GetScriptForDestination(destination); - CAmount nAmount = AmountFromValue(outputs[name_]); - - CTxOut out(nAmount, scriptPubKey); - rawTx.vout.push_back(out); - } - } - - if (!rbf.isNull() && rawTx.vin.size() > 0 && rbfOptIn != SignalsOptInRBF(CTransaction(rawTx))) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter combination: Sequence number(s) contradict replaceable option"); - } - - return rawTx; -} - static UniValue createrawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) { @@ -608,33 +505,54 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) return result; } +static std::string GetAllOutputTypes() +{ + std::string ret; + for (int i = TX_NONSTANDARD; i <= TX_WITNESS_UNKNOWN; ++i) { + if (i != TX_NONSTANDARD) ret += ", "; + ret += GetTxnOutputType(static_cast<txnouttype>(i)); + } + return ret; +} + static UniValue decodescript(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( - RPCHelpMan{"decodescript", + const RPCHelpMan help{"decodescript", "\nDecode a hex-encoded script.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, }, RPCResult{ "{\n" - " \"asm\":\"asm\", (string) Script public key\n" - " \"hex\":\"hex\", (string) hex-encoded public key\n" - " \"type\":\"type\", (string) The output type\n" - " \"reqSigs\": n, (numeric) The required signatures\n" - " \"addresses\": [ (json array of string)\n" - " \"address\" (string) bitcoin address\n" + " \"asm\":\"asm\", (string) Script public key\n" + " \"type\":\"type\", (string) The output type (e.g. "+GetAllOutputTypes()+")\n" + " \"reqSigs\": n, (numeric) The required signatures\n" + " \"addresses\": [ (json array of string)\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ],\n" - " \"p2sh\",\"address\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"p2sh\":\"str\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"segwit\": { (json object) Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness).\n" + " \"asm\":\"str\", (string) String representation of the script public key\n" + " \"hex\":\"hexstr\", (string) Hex string of the script public key\n" + " \"type\":\"str\", (string) The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)\n" + " \"reqSigs\": n, (numeric) The required signatures (always 1)\n" + " \"addresses\": [ (json array of string) (always length 1)\n" + " \"address\" (string) segwit address\n" + " ,...\n" + " ],\n" + " \"p2sh-segwit\":\"str\" (string) address of the P2SH script wrapping this witness redeem script.\n" "}\n" }, RPCExamples{ HelpExampleCli("decodescript", "\"hexstring\"") + HelpExampleRpc("decodescript", "\"hexstring\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } RPCTypeCheck(request.params, {UniValue::VSTR}); @@ -646,7 +564,7 @@ static UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, false); + ScriptPubKeyToUniv(script, r, /* fIncludeHex */ false); UniValue type; type = find_value(r, "type"); @@ -680,7 +598,7 @@ static UniValue decodescript(const JSONRPCRequest& request) // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, true); + ScriptPubKeyToUniv(segwitScr, sr, /* fIncludeHex */ true); sr.pushKV("p2sh-segwit", EncodeDestination(CScriptID(segwitScr))); r.pushKV("segwit", sr); } @@ -689,23 +607,6 @@ static UniValue decodescript(const JSONRPCRequest& request) return r; } -/** Pushes a JSON object for script verification or signing errors to vErrorsRet. */ -static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std::string& strMessage) -{ - UniValue entry(UniValue::VOBJ); - entry.pushKV("txid", txin.prevout.hash.ToString()); - entry.pushKV("vout", (uint64_t)txin.prevout.n); - UniValue witness(UniValue::VARR); - for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { - witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); - } - entry.pushKV("witness", witness); - entry.pushKV("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); - entry.pushKV("sequence", (uint64_t)txin.nSequence); - entry.pushKV("error", strMessage); - vErrorsRet.push_back(entry); -} - static UniValue combinerawtransaction(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) @@ -790,155 +691,6 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) return EncodeHexTx(CTransaction(mergedTx)); } -UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, const UniValue& prevTxsUnival, CBasicKeyStore *keystore, bool is_temp_keystore, const UniValue& hashType) -{ - // Fetch previous transactions (inputs): - CCoinsView viewDummy; - CCoinsViewCache view(&viewDummy); - { - LOCK2(cs_main, mempool.cs); - CCoinsViewCache &viewChain = *pcoinsTip; - CCoinsViewMemPool viewMempool(&viewChain, mempool); - view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view - - for (const CTxIn& txin : mtx.vin) { - view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. - } - - view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long - } - - // Add previous txouts given in the RPC call: - if (!prevTxsUnival.isNull()) { - UniValue prevTxs = prevTxsUnival.get_array(); - for (unsigned int idx = 0; idx < prevTxs.size(); ++idx) { - const UniValue& p = prevTxs[idx]; - if (!p.isObject()) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "expected object with {\"txid'\",\"vout\",\"scriptPubKey\"}"); - } - - UniValue prevOut = p.get_obj(); - - RPCTypeCheckObj(prevOut, - { - {"txid", UniValueType(UniValue::VSTR)}, - {"vout", UniValueType(UniValue::VNUM)}, - {"scriptPubKey", UniValueType(UniValue::VSTR)}, - }); - - uint256 txid = ParseHashO(prevOut, "txid"); - - int nOut = find_value(prevOut, "vout").get_int(); - if (nOut < 0) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); - } - - COutPoint out(txid, nOut); - std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); - CScript scriptPubKey(pkData.begin(), pkData.end()); - - { - const Coin& coin = view.AccessCoin(out); - if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { - std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ - ScriptToAsmStr(scriptPubKey); - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); - } - Coin newcoin; - newcoin.out.scriptPubKey = scriptPubKey; - newcoin.out.nValue = MAX_MONEY; - if (prevOut.exists("amount")) { - newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); - } - newcoin.nHeight = 1; - view.AddCoin(out, std::move(newcoin), true); - } - - // if redeemScript and private keys were given, add redeemScript to the keystore so it can be signed - if (is_temp_keystore && (scriptPubKey.IsPayToScriptHash() || scriptPubKey.IsPayToWitnessScriptHash())) { - RPCTypeCheckObj(prevOut, - { - {"redeemScript", UniValueType(UniValue::VSTR)}, - {"witnessScript", UniValueType(UniValue::VSTR)}, - }, true); - UniValue rs = find_value(prevOut, "redeemScript"); - if (!rs.isNull()) { - std::vector<unsigned char> rsData(ParseHexV(rs, "redeemScript")); - CScript redeemScript(rsData.begin(), rsData.end()); - keystore->AddCScript(redeemScript); - // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). - // This is only for compatibility, it is encouraged to use the explicit witnessScript field instead. - keystore->AddCScript(GetScriptForWitness(redeemScript)); - } - UniValue ws = find_value(prevOut, "witnessScript"); - if (!ws.isNull()) { - std::vector<unsigned char> wsData(ParseHexV(ws, "witnessScript")); - CScript witnessScript(wsData.begin(), wsData.end()); - keystore->AddCScript(witnessScript); - // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). - keystore->AddCScript(GetScriptForWitness(witnessScript)); - } - } - } - } - - int nHashType = ParseSighashString(hashType); - - bool fHashSingle = ((nHashType & ~SIGHASH_ANYONECANPAY) == SIGHASH_SINGLE); - - // Script verification errors - UniValue vErrors(UniValue::VARR); - - // Use CTransaction for the constant parts of the - // transaction to avoid rehashing. - const CTransaction txConst(mtx); - // Sign what we can: - for (unsigned int i = 0; i < mtx.vin.size(); i++) { - CTxIn& txin = mtx.vin[i]; - const Coin& coin = view.AccessCoin(txin.prevout); - if (coin.IsSpent()) { - TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); - continue; - } - const CScript& prevPubKey = coin.out.scriptPubKey; - const CAmount& amount = coin.out.nValue; - - SignatureData sigdata = DataFromTransaction(mtx, i, coin.out); - // Only sign SIGHASH_SINGLE if there's a corresponding output: - if (!fHashSingle || (i < mtx.vout.size())) { - ProduceSignature(*keystore, MutableTransactionSignatureCreator(&mtx, i, amount, nHashType), prevPubKey, sigdata); - } - - UpdateInput(txin, sigdata); - - // amount must be specified for valid segwit signature - if (amount == MAX_MONEY && !txin.scriptWitness.IsNull()) { - throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing amount for %s", coin.out.ToString())); - } - - ScriptError serror = SCRIPT_ERR_OK; - if (!VerifyScript(txin.scriptSig, prevPubKey, &txin.scriptWitness, STANDARD_SCRIPT_VERIFY_FLAGS, TransactionSignatureChecker(&txConst, i, amount), &serror)) { - if (serror == SCRIPT_ERR_INVALID_STACK_OPERATION) { - // Unable to sign input and verification failed (possible attempt to partially sign). - TxInErrorToJSON(txin, vErrors, "Unable to sign input, invalid stack size (possibly missing key)"); - } else { - TxInErrorToJSON(txin, vErrors, ScriptErrorString(serror)); - } - } - } - bool fComplete = vErrors.empty(); - - UniValue result(UniValue::VOBJ); - result.pushKV("hex", EncodeHexTx(CTransaction(mtx))); - result.pushKV("complete", fComplete); - if (!vErrors.empty()) { - result.pushKV("errors", vErrors); - } - - return result; -} - static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 4) @@ -965,7 +717,7 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -1019,27 +771,26 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) keystore.AddKey(key); } - return SignTransaction(*g_rpc_interfaces->chain, mtx, request.params[2], &keystore, true, request.params[3]); -} + // Fetch previous transactions (inputs): + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : mtx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + FindCoins(coins); -UniValue signrawtransaction(const JSONRPCRequest& request) -{ - // This method should be removed entirely in V0.19, along with the entries in the - // CRPCCommand table and rpc/client.cpp. - throw JSONRPCError(RPC_METHOD_DEPRECATED, "signrawtransaction was removed in v0.18.\n" - "Clients should transition to using signrawtransactionwithkey and signrawtransactionwithwallet"); + return SignTransaction(mtx, request.params[2], &keystore, coins, true, request.params[3]); } static UniValue sendrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( - RPCHelpMan{"sendrawtransaction", + const RPCHelpMan help{"sendrawtransaction", "\nSubmits raw transaction (serialized, hex-encoded) to local node and network.\n" "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), + "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + + "/kB.\nSet to 0 to accept any fee rate.\n"}, }, RPCResult{ "\"hex\" (string) The transaction hash in hex\n" @@ -1054,9 +805,16 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") }, - }.ToString()); + }; - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } + + RPCTypeCheck(request.params, { + UniValue::VSTR, + UniValueType(), // NUM or BOOL, checked later + }); // parse hex string from parameter CMutableTransaction mtx; @@ -1064,12 +822,22 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - bool allowhighfees = false; - if (!request.params[1].isNull()) allowhighfees = request.params[1].get_bool(); - const CAmount highfee{allowhighfees ? 0 : ::maxTxFee}; + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; + // TODO: temporary migration code for old clients. Remove in v0.20 + if (request.params[1].isBool()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); + } else if (!request.params[1].isNull()) { + size_t weight = GetTransactionWeight(*tx); + CFeeRate fr(AmountFromValue(request.params[1])); + // the +3/4 part rounds the value up, and is the same formula used when + // calculating the fee for a transaction + // (see GetVirtualTransactionSize) + max_raw_tx_fee = fr.GetFee((weight+3)/4); + } + uint256 txid; std::string err_string; - const TransactionError err = BroadcastTransaction(tx, txid, err_string, highfee); + const TransactionError err = BroadcastTransaction(tx, txid, err_string, max_raw_tx_fee); if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -1079,9 +847,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) static UniValue testmempoolaccept(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"testmempoolaccept", + const RPCHelpMan help{"testmempoolaccept", "\nReturns result of mempool acceptance tests indicating if raw transaction (serialized, hex-encoded) would be accepted by mempool.\n" "\nThis checks if the transaction violates the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", @@ -1092,7 +858,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, }, RPCResult{ "[ (array) The result of the mempool acceptance test for each raw transaction in the input array.\n" @@ -1114,10 +880,17 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); } - RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VBOOL}); + RPCTypeCheck(request.params, { + UniValue::VARR, + UniValueType(), // NUM or BOOL, checked later + }); + if (request.params[0].get_array().size() != 1) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Array must contain exactly one raw transaction for now"); } @@ -1129,9 +902,17 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CAmount max_raw_tx_fee = ::maxTxFee; - if (!request.params[1].isNull() && request.params[1].get_bool()) { - max_raw_tx_fee = 0; + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; + // TODO: temporary migration code for old clients. Remove in v0.20 + if (request.params[1].isBool()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); + } else if (!request.params[1].isNull()) { + size_t weight = GetTransactionWeight(*tx); + CFeeRate fr(AmountFromValue(request.params[1])); + // the +3/4 part rounds the value up, and is the same formula used when + // calculating the fee for a transaction + // (see GetVirtualTransactionSize) + max_raw_tx_fee = fr.GetFee((weight+3)/4); } UniValue result(UniValue::VARR); @@ -1500,7 +1281,6 @@ UniValue combinepsbt(const JSONRPCRequest& request) throw JSONRPCTransactionError(error); } - UniValue result(UniValue::VOBJ); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); @@ -1862,21 +1642,21 @@ UniValue analyzepsbt(const JSONRPCRequest& request) " \"has_utxo\" : true|false (boolean) Whether a UTXO is provided\n" " \"is_final\" : true|false (boolean) Whether the input is finalized\n" " \"missing\" : { (json object, optional) Things that are missing that are required to complete this input\n" - " \"pubkeys\" : [ (array)\n" + " \"pubkeys\" : [ (array, optional)\n" " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose BIP 32 derivation path is missing\n" " ]\n" - " \"signatures\" : [ (array)\n" + " \"signatures\" : [ (array, optional)\n" " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose signature is missing\n" " ]\n" - " \"redeemscript\" : \"hash\" (string) Hash160 of the redeemScript that is missing\n" - " \"witnessscript\" : \"hash\" (string) SHA256 of the witnessScript that is missing\n" + " \"redeemscript\" : \"hash\" (string, optional) Hash160 of the redeemScript that is missing\n" + " \"witnessscript\" : \"hash\" (string, optional) SHA256 of the witnessScript that is missing\n" " }\n" - " \"next\" : \"role\" (string) Role of the next person that this input needs to go to\n" + " \"next\" : \"role\" (string, optional) Role of the next person that this input needs to go to\n" " }\n" " ,...\n" " ]\n" - " \"estimated_vsize\" : vsize (numeric) Estimated vsize of the final signed transaction\n" - " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"estimated_vsize\" : vsize (numeric, optional) Estimated vsize of the final signed transaction\n" + " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction in " + CURRENCY_UNIT + "/kB. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"fee\" : fee (numeric, optional) The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"next\" : \"role\" (string) Role of the next person that this psbt needs to go to\n" "}\n" @@ -1895,148 +1675,56 @@ UniValue analyzepsbt(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } - // Go through each input and build status + PSBTAnalysis psbta = AnalyzePSBT(psbtx); + UniValue result(UniValue::VOBJ); UniValue inputs_result(UniValue::VARR); - bool calc_fee = true; - bool all_final = true; - bool only_missing_sigs = true; - bool only_missing_final = false; - CAmount in_amt = 0; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; + for (const auto& input : psbta.inputs) { UniValue input_univ(UniValue::VOBJ); UniValue missing(UniValue::VOBJ); - // Check for a UTXO - CTxOut utxo; - if (psbtx.GetInputUTXO(utxo, i)) { - in_amt += utxo.nValue; - input_univ.pushKV("has_utxo", true); - } else { - input_univ.pushKV("has_utxo", false); - input_univ.pushKV("is_final", false); - input_univ.pushKV("next", "updater"); - calc_fee = false; - } + input_univ.pushKV("has_utxo", input.has_utxo); + input_univ.pushKV("is_final", input.is_final); + input_univ.pushKV("next", PSBTRoleName(input.next)); - // Check if it is final - if (!utxo.IsNull() && !PSBTInputSigned(input)) { - input_univ.pushKV("is_final", false); - all_final = false; - - // Figure out what is missing - SignatureData outdata; - bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); - - // Things are missing - if (!complete) { - if (!outdata.missing_pubkeys.empty()) { - // Missing pubkeys - UniValue missing_pubkeys_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_pubkeys) { - missing_pubkeys_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("pubkeys", missing_pubkeys_univ); - } - if (!outdata.missing_redeem_script.IsNull()) { - // Missing redeemScript - missing.pushKV("redeemscript", HexStr(outdata.missing_redeem_script)); - } - if (!outdata.missing_witness_script.IsNull()) { - // Missing witnessScript - missing.pushKV("witnessscript", HexStr(outdata.missing_witness_script)); - } - if (!outdata.missing_sigs.empty()) { - // Missing sigs - UniValue missing_sigs_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_sigs) { - missing_sigs_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("signatures", missing_sigs_univ); - } - input_univ.pushKV("missing", missing); - - // If we are only missing signatures and nothing else, then next is signer - if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { - input_univ.pushKV("next", "signer"); - } else { - only_missing_sigs = false; - input_univ.pushKV("next", "updater"); - } - } else { - only_missing_final = true; - input_univ.pushKV("next", "finalizer"); + if (!input.missing_pubkeys.empty()) { + UniValue missing_pubkeys_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_pubkeys) { + missing_pubkeys_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("pubkeys", missing_pubkeys_univ); + } + if (!input.missing_redeem_script.IsNull()) { + missing.pushKV("redeemscript", HexStr(input.missing_redeem_script)); + } + if (!input.missing_witness_script.IsNull()) { + missing.pushKV("witnessscript", HexStr(input.missing_witness_script)); + } + if (!input.missing_sigs.empty()) { + UniValue missing_sigs_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_sigs) { + missing_sigs_univ.push_back(HexStr(pubkey)); } - } else if (!utxo.IsNull()){ - input_univ.pushKV("is_final", true); + missing.pushKV("signatures", missing_sigs_univ); + } + if (!missing.getKeys().empty()) { + input_univ.pushKV("missing", missing); } inputs_result.push_back(input_univ); } result.pushKV("inputs", inputs_result); - if (all_final) { - only_missing_sigs = false; - result.pushKV("next", "extractor"); + if (psbta.estimated_vsize != nullopt) { + result.pushKV("estimated_vsize", (int)*psbta.estimated_vsize); } - if (calc_fee) { - // Get the output amount - CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), 0, - [](int a, const CTxOut& b) { - return a += b.nValue; - } - ); - - // Get the fee - CAmount fee = in_amt - out_amt; - - // Estimate the size - CMutableTransaction mtx(*psbtx.tx); - CCoinsView view_dummy; - CCoinsViewCache view(&view_dummy); - bool success = true; - - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; - if (SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true)) { - mtx.vin[i].scriptSig = input.final_script_sig; - mtx.vin[i].scriptWitness = input.final_script_witness; - - Coin newcoin; - if (!psbtx.GetInputUTXO(newcoin.out, i)) { - success = false; - break; - } - newcoin.nHeight = 1; - view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); - } else { - success = false; - break; - } - } - - if (success) { - CTransaction ctx = CTransaction(mtx); - size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); - result.pushKV("estimated_vsize", (int)size); - // Estimate fee rate - CFeeRate feerate(fee, size); - result.pushKV("estimated_feerate", feerate.ToString()); - } - result.pushKV("fee", ValueFromAmount(fee)); - - if (only_missing_sigs) { - result.pushKV("next", "signer"); - } else if (only_missing_final) { - result.pushKV("next", "finalizer"); - } else if (all_final) { - result.pushKV("next", "extractor"); - } else { - result.pushKV("next", "updater"); - } - } else { - result.pushKV("next", "updater"); + if (psbta.estimated_feerate != nullopt) { + result.pushKV("estimated_feerate", ValueFromAmount(psbta.estimated_feerate->GetFeePerK())); } + if (psbta.fee != nullopt) { + result.pushKV("fee", ValueFromAmount(*psbta.fee)); + } + result.pushKV("next", PSBTRoleName(psbta.next)); + return result; } @@ -2048,11 +1736,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, - { "hidden", "signrawtransaction", &signrawtransaction, {"hexstring","prevtxs","privkeys","sighashtype"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, diff --git a/src/rpc/rawtransaction.h b/src/rpc/rawtransaction.h deleted file mode 100644 index 52d701d1c3..0000000000 --- a/src/rpc/rawtransaction.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_RPC_RAWTRANSACTION_H -#define BITCOIN_RPC_RAWTRANSACTION_H - -class CBasicKeyStore; -struct CMutableTransaction; -class UniValue; - -namespace interfaces { -class Chain; -} // namespace interfaces - -/** Sign a transaction with the given keystore and previous transactions */ -UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, const UniValue& prevTxs, CBasicKeyStore *keystore, bool tempKeystore, const UniValue& hashType); - -/** Create a transaction from univalue parameters */ -CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, const UniValue& rbf); - -#endif // BITCOIN_RPC_RAWTRANSACTION_H diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp new file mode 100644 index 0000000000..8af491977c --- /dev/null +++ b/src/rpc/rawtransaction_util.cpp @@ -0,0 +1,283 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/rawtransaction_util.h> + +#include <coins.h> +#include <core_io.h> +#include <interfaces/chain.h> +#include <key_io.h> +#include <keystore.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <rpc/protocol.h> +#include <rpc/util.h> +#include <tinyformat.h> +#include <univalue.h> +#include <util/rbf.h> +#include <util/strencodings.h> + +CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, const UniValue& rbf) +{ + if (inputs_in.isNull() || outputs_in.isNull()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); + + UniValue inputs = inputs_in.get_array(); + const bool outputs_is_obj = outputs_in.isObject(); + UniValue outputs = outputs_is_obj ? outputs_in.get_obj() : outputs_in.get_array(); + + CMutableTransaction rawTx; + + if (!locktime.isNull()) { + int64_t nLockTime = locktime.get_int64(); + if (nLockTime < 0 || nLockTime > LOCKTIME_MAX) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, locktime out of range"); + rawTx.nLockTime = nLockTime; + } + + bool rbfOptIn = rbf.isTrue(); + + for (unsigned int idx = 0; idx < inputs.size(); idx++) { + const UniValue& input = inputs[idx]; + const UniValue& o = input.get_obj(); + + uint256 txid = ParseHashO(o, "txid"); + + const UniValue& vout_v = find_value(o, "vout"); + if (!vout_v.isNum()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); + int nOutput = vout_v.get_int(); + if (nOutput < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); + + uint32_t nSequence; + if (rbfOptIn) { + nSequence = MAX_BIP125_RBF_SEQUENCE; /* CTxIn::SEQUENCE_FINAL - 2 */ + } else if (rawTx.nLockTime) { + nSequence = CTxIn::SEQUENCE_FINAL - 1; + } else { + nSequence = CTxIn::SEQUENCE_FINAL; + } + + // set the sequence number if passed in the parameters object + const UniValue& sequenceObj = find_value(o, "sequence"); + if (sequenceObj.isNum()) { + int64_t seqNr64 = sequenceObj.get_int64(); + if (seqNr64 < 0 || seqNr64 > CTxIn::SEQUENCE_FINAL) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, sequence number is out of range"); + } else { + nSequence = (uint32_t)seqNr64; + } + } + + CTxIn in(COutPoint(txid, nOutput), CScript(), nSequence); + + rawTx.vin.push_back(in); + } + + if (!outputs_is_obj) { + // Translate array of key-value pairs into dict + UniValue outputs_dict = UniValue(UniValue::VOBJ); + for (size_t i = 0; i < outputs.size(); ++i) { + const UniValue& output = outputs[i]; + if (!output.isObject()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, key-value pair not an object as expected"); + } + if (output.size() != 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, key-value pair must contain exactly one key"); + } + outputs_dict.pushKVs(output); + } + outputs = std::move(outputs_dict); + } + + // Duplicate checking + std::set<CTxDestination> destinations; + bool has_data{false}; + + for (const std::string& name_ : outputs.getKeys()) { + if (name_ == "data") { + if (has_data) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, duplicate key: data"); + } + has_data = true; + std::vector<unsigned char> data = ParseHexV(outputs[name_].getValStr(), "Data"); + + CTxOut out(0, CScript() << OP_RETURN << data); + rawTx.vout.push_back(out); + } else { + CTxDestination destination = DecodeDestination(name_); + if (!IsValidDestination(destination)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string("Invalid Bitcoin address: ") + name_); + } + + if (!destinations.insert(destination).second) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated address: ") + name_); + } + + CScript scriptPubKey = GetScriptForDestination(destination); + CAmount nAmount = AmountFromValue(outputs[name_]); + + CTxOut out(nAmount, scriptPubKey); + rawTx.vout.push_back(out); + } + } + + if (!rbf.isNull() && rawTx.vin.size() > 0 && rbfOptIn != SignalsOptInRBF(CTransaction(rawTx))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter combination: Sequence number(s) contradict replaceable option"); + } + + return rawTx; +} + +/** Pushes a JSON object for script verification or signing errors to vErrorsRet. */ +static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std::string& strMessage) +{ + UniValue entry(UniValue::VOBJ); + entry.pushKV("txid", txin.prevout.hash.ToString()); + entry.pushKV("vout", (uint64_t)txin.prevout.n); + UniValue witness(UniValue::VARR); + for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { + witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + } + entry.pushKV("witness", witness); + entry.pushKV("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + entry.pushKV("sequence", (uint64_t)txin.nSequence); + entry.pushKV("error", strMessage); + vErrorsRet.push_back(entry); +} + +UniValue SignTransaction(CMutableTransaction& mtx, const UniValue& prevTxsUnival, CBasicKeyStore* keystore, std::map<COutPoint, Coin>& coins, bool is_temp_keystore, const UniValue& hashType) +{ + // Add previous txouts given in the RPC call: + if (!prevTxsUnival.isNull()) { + UniValue prevTxs = prevTxsUnival.get_array(); + for (unsigned int idx = 0; idx < prevTxs.size(); ++idx) { + const UniValue& p = prevTxs[idx]; + if (!p.isObject()) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "expected object with {\"txid'\",\"vout\",\"scriptPubKey\"}"); + } + + UniValue prevOut = p.get_obj(); + + RPCTypeCheckObj(prevOut, + { + {"txid", UniValueType(UniValue::VSTR)}, + {"vout", UniValueType(UniValue::VNUM)}, + {"scriptPubKey", UniValueType(UniValue::VSTR)}, + }); + + uint256 txid = ParseHashO(prevOut, "txid"); + + int nOut = find_value(prevOut, "vout").get_int(); + if (nOut < 0) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); + } + + COutPoint out(txid, nOut); + std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); + CScript scriptPubKey(pkData.begin(), pkData.end()); + + { + auto coin = coins.find(out); + if (coin != coins.end() && !coin->second.IsSpent() && coin->second.out.scriptPubKey != scriptPubKey) { + std::string err("Previous output scriptPubKey mismatch:\n"); + err = err + ScriptToAsmStr(coin->second.out.scriptPubKey) + "\nvs:\n"+ + ScriptToAsmStr(scriptPubKey); + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); + } + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = MAX_MONEY; + if (prevOut.exists("amount")) { + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); + } + newcoin.nHeight = 1; + coins[out] = std::move(newcoin); + } + + // if redeemScript and private keys were given, add redeemScript to the keystore so it can be signed + if (is_temp_keystore && (scriptPubKey.IsPayToScriptHash() || scriptPubKey.IsPayToWitnessScriptHash())) { + RPCTypeCheckObj(prevOut, + { + {"redeemScript", UniValueType(UniValue::VSTR)}, + {"witnessScript", UniValueType(UniValue::VSTR)}, + }, true); + UniValue rs = find_value(prevOut, "redeemScript"); + if (!rs.isNull()) { + std::vector<unsigned char> rsData(ParseHexV(rs, "redeemScript")); + CScript redeemScript(rsData.begin(), rsData.end()); + keystore->AddCScript(redeemScript); + // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + // This is only for compatibility, it is encouraged to use the explicit witnessScript field instead. + keystore->AddCScript(GetScriptForWitness(redeemScript)); + } + UniValue ws = find_value(prevOut, "witnessScript"); + if (!ws.isNull()) { + std::vector<unsigned char> wsData(ParseHexV(ws, "witnessScript")); + CScript witnessScript(wsData.begin(), wsData.end()); + keystore->AddCScript(witnessScript); + // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + keystore->AddCScript(GetScriptForWitness(witnessScript)); + } + } + } + } + + int nHashType = ParseSighashString(hashType); + + bool fHashSingle = ((nHashType & ~SIGHASH_ANYONECANPAY) == SIGHASH_SINGLE); + + // Script verification errors + UniValue vErrors(UniValue::VARR); + + // Use CTransaction for the constant parts of the + // transaction to avoid rehashing. + const CTransaction txConst(mtx); + // Sign what we can: + for (unsigned int i = 0; i < mtx.vin.size(); i++) { + CTxIn& txin = mtx.vin[i]; + auto coin = coins.find(txin.prevout); + if (coin == coins.end() || coin->second.IsSpent()) { + TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); + continue; + } + const CScript& prevPubKey = coin->second.out.scriptPubKey; + const CAmount& amount = coin->second.out.nValue; + + SignatureData sigdata = DataFromTransaction(mtx, i, coin->second.out); + // Only sign SIGHASH_SINGLE if there's a corresponding output: + if (!fHashSingle || (i < mtx.vout.size())) { + ProduceSignature(*keystore, MutableTransactionSignatureCreator(&mtx, i, amount, nHashType), prevPubKey, sigdata); + } + + UpdateInput(txin, sigdata); + + // amount must be specified for valid segwit signature + if (amount == MAX_MONEY && !txin.scriptWitness.IsNull()) { + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing amount for %s", coin->second.out.ToString())); + } + + ScriptError serror = SCRIPT_ERR_OK; + if (!VerifyScript(txin.scriptSig, prevPubKey, &txin.scriptWitness, STANDARD_SCRIPT_VERIFY_FLAGS, TransactionSignatureChecker(&txConst, i, amount), &serror)) { + if (serror == SCRIPT_ERR_INVALID_STACK_OPERATION) { + // Unable to sign input and verification failed (possible attempt to partially sign). + TxInErrorToJSON(txin, vErrors, "Unable to sign input, invalid stack size (possibly missing key)"); + } else { + TxInErrorToJSON(txin, vErrors, ScriptErrorString(serror)); + } + } + } + bool fComplete = vErrors.empty(); + + UniValue result(UniValue::VOBJ); + result.pushKV("hex", EncodeHexTx(CTransaction(mtx))); + result.pushKV("complete", fComplete); + if (!vErrors.empty()) { + result.pushKV("errors", vErrors); + } + + return result; +} diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h new file mode 100644 index 0000000000..c115d33a77 --- /dev/null +++ b/src/rpc/rawtransaction_util.h @@ -0,0 +1,32 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_RAWTRANSACTION_UTIL_H +#define BITCOIN_RPC_RAWTRANSACTION_UTIL_H + +#include <map> + +class CBasicKeyStore; +class UniValue; +struct CMutableTransaction; +class Coin; +class COutPoint; + +/** + * Sign a transaction with the given keystore and previous transactions + * + * @param mtx The transaction to-be-signed + * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain + * @param keystore Temporary keystore containing signing keys + * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call + * @param tempKeystore Whether to use temporary keystore + * @param hashType The signature hash type + * @returns JSON object with details of signed transaction + */ +UniValue SignTransaction(CMutableTransaction& mtx, const UniValue& prevTxs, CBasicKeyStore* keystore, std::map<COutPoint, Coin>& coins, bool tempKeystore, const UniValue& hashType); + +/** Create a transaction from univalue parameters */ +CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, const UniValue& rbf); + +#endif // BITCOIN_RPC_RAWTRANSACTION_UTIL_H diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index cd90573da0..9df4070cbb 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -30,6 +30,7 @@ static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server starte static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers; +static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler); struct RPCCommandExecutionInfo { @@ -76,108 +77,15 @@ void RPCServer::OnStopped(std::function<void ()> slot) g_rpcSignals.Stopped.connect(slot); } -void RPCTypeCheck(const UniValue& params, - const std::list<UniValueType>& typesExpected, - bool fAllowNull) -{ - unsigned int i = 0; - for (const UniValueType& t : typesExpected) { - if (params.size() <= i) - break; - - const UniValue& v = params[i]; - if (!(fAllowNull && v.isNull())) { - RPCTypeCheckArgument(v, t); - } - i++; - } -} - -void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected) -{ - if (!typeExpected.typeAny && value.type() != typeExpected.type) { - throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Expected type %s, got %s", uvTypeName(typeExpected.type), uvTypeName(value.type()))); - } -} - -void RPCTypeCheckObj(const UniValue& o, - const std::map<std::string, UniValueType>& typesExpected, - bool fAllowNull, - bool fStrict) -{ - for (const auto& t : typesExpected) { - const UniValue& v = find_value(o, t.first); - if (!fAllowNull && v.isNull()) - throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); - - if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) { - std::string err = strprintf("Expected type %s for %s, got %s", - uvTypeName(t.second.type), t.first, uvTypeName(v.type())); - throw JSONRPCError(RPC_TYPE_ERROR, err); - } - } - - if (fStrict) - { - for (const std::string& k : o.getKeys()) - { - if (typesExpected.count(k) == 0) - { - std::string err = strprintf("Unexpected key %s", k); - throw JSONRPCError(RPC_TYPE_ERROR, err); - } - } - } -} - -CAmount AmountFromValue(const UniValue& value) -{ - if (!value.isNum() && !value.isStr()) - throw JSONRPCError(RPC_TYPE_ERROR, "Amount is not a number or string"); - CAmount amount; - if (!ParseFixedPoint(value.getValStr(), 8, &amount)) - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount"); - if (!MoneyRange(amount)) - throw JSONRPCError(RPC_TYPE_ERROR, "Amount out of range"); - return amount; -} - -uint256 ParseHashV(const UniValue& v, std::string strName) -{ - std::string strHex(v.get_str()); - if (64 != strHex.length()) - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", strName, 64, strHex.length(), strHex)); - if (!IsHex(strHex)) // Note: IsHex("") is false - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); - return uint256S(strHex); -} -uint256 ParseHashO(const UniValue& o, std::string strKey) -{ - return ParseHashV(find_value(o, strKey), strKey); -} -std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) -{ - std::string strHex; - if (v.isStr()) - strHex = v.get_str(); - if (!IsHex(strHex)) - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); - return ParseHex(strHex); -} -std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) -{ - return ParseHexV(find_value(o, strKey), strKey); -} - std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& helpreq) const { std::string strRet; std::string category; - std::set<rpcfn_type> setDone; + std::set<intptr_t> setDone; std::vector<std::pair<std::string, const CRPCCommand*> > vCommands; for (const auto& entry : mapCommands) - vCommands.push_back(make_pair(entry.second->category + entry.first, entry.second)); + vCommands.push_back(make_pair(entry.second.front()->category + entry.first, entry.second.front())); sort(vCommands.begin(), vCommands.end()); JSONRPCRequest jreq(helpreq); @@ -193,9 +101,9 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& jreq.strMethod = strMethod; try { - rpcfn_type pfn = pcmd->actor; - if (setDone.insert(pfn).second) - (*pfn)(jreq); + UniValue unused_result; + if (setDone.insert(pcmd->unique_id).second) + pcmd->actor(jreq, unused_result, true /* last_handler */); } catch (const std::exception& e) { @@ -296,8 +204,20 @@ static UniValue getrpcinfo(const JSONRPCRequest& request) RPCHelpMan{"getrpcinfo", "\nReturns details of the RPC server.\n", {}, - RPCResults{}, - RPCExamples{""}, + RPCResult{ + "{\n" + " \"active_commands\" (array) All active commands\n" + " [\n" + " { (object) Information about an active command\n" + " \"method\" (string) The name of the RPC command \n" + " \"duration\" (numeric) The running time in microseconds\n" + " },...\n" + " ]\n" + "}\n" + }, + RPCExamples{ + HelpExampleCli("getrpcinfo", "") + + HelpExampleRpc("getrpcinfo", "")}, }.ToString() ); } @@ -337,32 +257,32 @@ CRPCTable::CRPCTable() const CRPCCommand *pcmd; pcmd = &vRPCCommands[vcidx]; - mapCommands[pcmd->name] = pcmd; + mapCommands[pcmd->name].push_back(pcmd); } } -const CRPCCommand *CRPCTable::operator[](const std::string &name) const -{ - std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); - if (it == mapCommands.end()) - return nullptr; - return (*it).second; -} - bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) { if (IsRPCRunning()) return false; - // don't allow overwriting for now - std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); - if (it != mapCommands.end()) - return false; - - mapCommands[name] = pcmd; + mapCommands[name].push_back(pcmd); return true; } +bool CRPCTable::removeCommand(const std::string& name, const CRPCCommand* pcmd) +{ + auto it = mapCommands.find(name); + if (it != mapCommands.end()) { + auto new_end = std::remove(it->second.begin(), it->second.end(), pcmd); + if (it->second.end() != new_end) { + it->second.erase(new_end, it->second.end()); + return true; + } + } + return false; +} + void StartRPC() { LogPrint(BCLog::RPC, "Starting RPC\n"); @@ -543,18 +463,28 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const } // Find method - const CRPCCommand *pcmd = tableRPC[request.strMethod]; - if (!pcmd) - throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found"); + auto it = mapCommands.find(request.strMethod); + if (it != mapCommands.end()) { + UniValue result; + for (const auto& command : it->second) { + if (ExecuteCommand(*command, request, result, &command == &it->second.back())) { + return result; + } + } + } + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found"); +} +static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler) +{ try { RPCCommandExecution execution(request.strMethod); // Execute, convert arguments to array if necessary if (request.params.isObject()) { - return pcmd->actor(transformNamedArguments(request, pcmd->argNames)); + return command.actor(transformNamedArguments(request, command.argNames), result, last_handler); } else { - return pcmd->actor(request); + return command.actor(request, result, last_handler); } } catch (const std::exception& e) @@ -570,17 +500,6 @@ std::vector<std::string> CRPCTable::listCommands() const return commandList; } -std::string HelpExampleCli(const std::string& methodname, const std::string& args) -{ - return "> bitcoin-cli " + methodname + " " + args + "\n"; -} - -std::string HelpExampleRpc(const std::string& methodname, const std::string& args) -{ - return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\":\"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": [" + args + "] }' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; -} - void RPCSetTimerInterfaceIfUnset(RPCTimerInterface *iface) { if (!timerInterface) diff --git a/src/rpc/server.h b/src/rpc/server.h index 2d62a76f3c..431ff0bb7c 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -27,15 +27,6 @@ namespace RPCServer void OnStopped(std::function<void ()> slot); } -/** Wrapper for UniValue::VType, which includes typeAny: - * Used to denote don't care type. */ -struct UniValueType { - UniValueType(UniValue::VType _type) : typeAny(false), type(_type) {} - UniValueType() : typeAny(true) {} - bool typeAny; - UniValue::VType type; -}; - class JSONRPCRequest { public: @@ -65,26 +56,6 @@ void SetRPCWarmupFinished(); /* returns the current warmup state. */ bool RPCIsInWarmup(std::string *outStatus); -/** - * Type-check arguments; throws JSONRPCError if wrong type given. Does not check that - * the right number of arguments are passed, just that any passed are the correct type. - */ -void RPCTypeCheck(const UniValue& params, - const std::list<UniValueType>& typesExpected, bool fAllowNull=false); - -/** - * Type-check one argument; throws JSONRPCError if wrong type given. - */ -void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected); - -/* - Check for expected keys/value types in an Object. -*/ -void RPCTypeCheckObj(const UniValue& o, - const std::map<std::string, UniValueType>& typesExpected, - bool fAllowNull = false, - bool fStrict = false); - /** Opaque base class for timers returned by NewTimerFunc. * This provides no methods at the moment, but makes sure that delete * cleans up the whole state. @@ -131,10 +102,31 @@ typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); class CRPCCommand { public: + //! RPC method handler reading request and assigning result. Should return + //! true if request is fully handled, false if it should be passed on to + //! subsequent handlers. + using Actor = std::function<bool(const JSONRPCRequest& request, UniValue& result, bool last_handler)>; + + //! Constructor taking Actor callback supporting multiple handlers. + CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::string> args, intptr_t unique_id) + : category(std::move(category)), name(std::move(name)), actor(std::move(actor)), argNames(std::move(args)), + unique_id(unique_id) + { + } + + //! Simplified constructor taking plain rpcfn_type function pointer. + CRPCCommand(const char* category, const char* name, rpcfn_type fn, std::initializer_list<const char*> args) + : CRPCCommand(category, name, + [fn](const JSONRPCRequest& request, UniValue& result, bool) { result = fn(request); return true; }, + {args.begin(), args.end()}, intptr_t(fn)) + { + } + std::string category; std::string name; - rpcfn_type actor; + Actor actor; std::vector<std::string> argNames; + intptr_t unique_id; }; /** @@ -143,10 +135,9 @@ public: class CRPCTable { private: - std::map<std::string, const CRPCCommand*> mapCommands; + std::map<std::string, std::vector<const CRPCCommand*>> mapCommands; public: CRPCTable(); - const CRPCCommand* operator[](const std::string& name) const; std::string help(const std::string& name, const JSONRPCRequest& helpreq) const; /** @@ -169,9 +160,7 @@ public: * * Returns false if RPC server is already running (dump concurrency protection). * - * Commands cannot be overwritten (returns false). - * - * Commands with different method names but the same callback function will + * Commands with different method names but the same unique_id will * be considered aliases, and only the first registered method name will * show up in the help text command listing. Aliased commands do not have * to have the same behavior. Server and client code can distinguish @@ -179,25 +168,13 @@ public: * register different names, types, and numbers of parameters. */ bool appendCommand(const std::string& name, const CRPCCommand* pcmd); + bool removeCommand(const std::string& name, const CRPCCommand* pcmd); }; bool IsDeprecatedRPCEnabled(const std::string& method); extern CRPCTable tableRPC; -/** - * Utilities: convert hex-encoded Values - * (throws error if not hex). - */ -extern uint256 ParseHashV(const UniValue& v, std::string strName); -extern uint256 ParseHashO(const UniValue& o, std::string strKey); -extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); -extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); - -extern CAmount AmountFromValue(const UniValue& value); -extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); -extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); - void StartRPC(); void InterruptRPC(); void StopRPC(); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 40ac133186..1a87c9f935 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -4,14 +4,116 @@ #include <key_io.h> #include <keystore.h> -#include <policy/fees.h> #include <rpc/util.h> #include <tinyformat.h> #include <util/strencodings.h> -#include <validation.h> InitInterfaces* g_rpc_interfaces = nullptr; +void RPCTypeCheck(const UniValue& params, + const std::list<UniValueType>& typesExpected, + bool fAllowNull) +{ + unsigned int i = 0; + for (const UniValueType& t : typesExpected) { + if (params.size() <= i) + break; + + const UniValue& v = params[i]; + if (!(fAllowNull && v.isNull())) { + RPCTypeCheckArgument(v, t); + } + i++; + } +} + +void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected) +{ + if (!typeExpected.typeAny && value.type() != typeExpected.type) { + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Expected type %s, got %s", uvTypeName(typeExpected.type), uvTypeName(value.type()))); + } +} + +void RPCTypeCheckObj(const UniValue& o, + const std::map<std::string, UniValueType>& typesExpected, + bool fAllowNull, + bool fStrict) +{ + for (const auto& t : typesExpected) { + const UniValue& v = find_value(o, t.first); + if (!fAllowNull && v.isNull()) + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); + + if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) { + std::string err = strprintf("Expected type %s for %s, got %s", + uvTypeName(t.second.type), t.first, uvTypeName(v.type())); + throw JSONRPCError(RPC_TYPE_ERROR, err); + } + } + + if (fStrict) + { + for (const std::string& k : o.getKeys()) + { + if (typesExpected.count(k) == 0) + { + std::string err = strprintf("Unexpected key %s", k); + throw JSONRPCError(RPC_TYPE_ERROR, err); + } + } + } +} + +CAmount AmountFromValue(const UniValue& value) +{ + if (!value.isNum() && !value.isStr()) + throw JSONRPCError(RPC_TYPE_ERROR, "Amount is not a number or string"); + CAmount amount; + if (!ParseFixedPoint(value.getValStr(), 8, &amount)) + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount"); + if (!MoneyRange(amount)) + throw JSONRPCError(RPC_TYPE_ERROR, "Amount out of range"); + return amount; +} + +uint256 ParseHashV(const UniValue& v, std::string strName) +{ + std::string strHex(v.get_str()); + if (64 != strHex.length()) + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", strName, 64, strHex.length(), strHex)); + if (!IsHex(strHex)) // Note: IsHex("") is false + throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + return uint256S(strHex); +} +uint256 ParseHashO(const UniValue& o, std::string strKey) +{ + return ParseHashV(find_value(o, strKey), strKey); +} +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) +{ + std::string strHex; + if (v.isStr()) + strHex = v.get_str(); + if (!IsHex(strHex)) + throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + return ParseHex(strHex); +} +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) +{ + return ParseHexV(find_value(o, strKey), strKey); +} + +std::string HelpExampleCli(const std::string& methodname, const std::string& args) +{ + return "> bitcoin-cli " + methodname + " " + args + "\n"; +} + +std::string HelpExampleRpc(const std::string& methodname, const std::string& args) +{ + return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\":\"curltest\", " + "\"method\": \"" + methodname + "\", \"params\": [" + args + "] }' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; +} + // Converts a hex string to a public key if possible CPubKey HexToPubKey(const std::string& hex_in) { @@ -130,10 +232,9 @@ UniValue DescribeAddress(const CTxDestination& dest) return boost::apply_visitor(DescribeAddressVisitor(), dest); } -unsigned int ParseConfirmTarget(const UniValue& value) +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) { int target = value.get_int(); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); if (target < 1 || (unsigned int)target > max_target) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); } @@ -168,6 +269,10 @@ UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_s } } +/** + * A pair of strings that can be aligned (through padding) with other Sections + * later on + */ struct Section { Section(const std::string& left, const std::string& right) : m_left{left}, m_right{right} {} @@ -175,6 +280,10 @@ struct Section { const std::string m_right; }; +/** + * Keeps track of RPCArgs by transforming them into sections for the purpose + * of serializing everything to a single string + */ struct Sections { std::vector<Section> m_sections; size_t m_max_pad{0}; @@ -185,16 +294,26 @@ struct Sections { m_sections.push_back(s); } + /** + * Serializing RPCArgs depends on the outer type. Only arrays and + * dictionaries can be nested in json. The top-level outer type is "named + * arguments", a mix between a dictionary and arrays. + */ enum class OuterType { ARR, OBJ, NAMED_ARG, // Only set on first recursion }; + /** + * Recursive helper to translate an RPCArg into sections + */ void Push(const RPCArg& arg, const size_t current_indent = 5, const OuterType outer_type = OuterType::NAMED_ARG) { const auto indent = std::string(current_indent, ' '); const auto indent_next = std::string(current_indent + 2, ' '); + const bool push_name{outer_type == OuterType::OBJ}; // Dictionary keys must have a name + switch (arg.m_type) { case RPCArg::Type::STR_HEX: case RPCArg::Type::STR: @@ -204,10 +323,10 @@ struct Sections { case RPCArg::Type::BOOL: { if (outer_type == OuterType::NAMED_ARG) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; - if (arg.m_type_str.size() != 0 && outer_type == OuterType::OBJ) { + if (arg.m_type_str.size() != 0 && push_name) { left += "\"" + arg.m_name + "\": " + arg.m_type_str.at(0); } else { - left += outer_type == OuterType::OBJ ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); + left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } left += ","; PushSection({left, arg.ToDescriptionString()}); @@ -216,7 +335,7 @@ struct Sections { case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); - PushSection({indent + "{", right}); + PushSection({indent + (push_name ? "\"" + arg.m_name + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); } @@ -228,7 +347,7 @@ struct Sections { } case RPCArg::Type::ARR: { auto left = indent; - left += outer_type == OuterType::OBJ ? "\"" + arg.m_name + "\": " : ""; + left += push_name ? "\"" + arg.m_name + "\": " : ""; left += "["; const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); PushSection({left, right}); @@ -244,6 +363,9 @@ struct Sections { } } + /** + * Concatenate all sections with proper padding + */ std::string ToString() const { std::string ret; diff --git a/src/rpc/util.h b/src/rpc/util.h index f1bd2c89df..b5b5789253 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -26,6 +26,48 @@ struct InitInterfaces; //! state to RPC method implementations. extern InitInterfaces* g_rpc_interfaces; +/** Wrapper for UniValue::VType, which includes typeAny: + * Used to denote don't care type. */ +struct UniValueType { + UniValueType(UniValue::VType _type) : typeAny(false), type(_type) {} + UniValueType() : typeAny(true) {} + bool typeAny; + UniValue::VType type; +}; + +/** + * Type-check arguments; throws JSONRPCError if wrong type given. Does not check that + * the right number of arguments are passed, just that any passed are the correct type. + */ +void RPCTypeCheck(const UniValue& params, + const std::list<UniValueType>& typesExpected, bool fAllowNull=false); + +/** + * Type-check one argument; throws JSONRPCError if wrong type given. + */ +void RPCTypeCheckArgument(const UniValue& value, const UniValueType& typeExpected); + +/* + Check for expected keys/value types in an Object. +*/ +void RPCTypeCheckObj(const UniValue& o, + const std::map<std::string, UniValueType>& typesExpected, + bool fAllowNull = false, + bool fStrict = false); + +/** + * Utilities: convert hex-encoded Values + * (throws error if not hex). + */ +extern uint256 ParseHashV(const UniValue& v, std::string strName); +extern uint256 ParseHashO(const UniValue& o, std::string strKey); +extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); +extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); + +extern CAmount AmountFromValue(const UniValue& value); +extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); +extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); + CPubKey HexToPubKey(const std::string& hex_in); CPubKey AddrToPubKey(CKeyStore* const keystore, const std::string& addr_in); CScript CreateMultisigRedeemscript(const int required, const std::vector<CPubKey>& pubkeys); @@ -33,7 +75,7 @@ CScript CreateMultisigRedeemscript(const int required, const std::vector<CPubKey UniValue DescribeAddress(const CTxDestination& dest); //! Parse a confirm target option and raise an RPC error if it is invalid. -unsigned int ParseConfirmTarget(const UniValue& value); +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); RPCErrorCode RPCErrorFromTransactionError(TransactionError terr); UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string = ""); diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 43448d7222..a333d4d4ac 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -436,7 +436,7 @@ public: pubkeys.reserve(entries.size()); for (auto& entry : entries) { pubkeys.push_back(entry.first); - out.origins.emplace(entry.first.GetID(), std::move(entry.second)); + out.origins.emplace(entry.first.GetID(), std::make_pair<CPubKey, KeyOriginInfo>(CPubKey(entry.first), std::move(entry.second))); } if (m_script_arg) { for (const auto& subscript : subscripts) { diff --git a/src/script/ismine.h b/src/script/ismine.h index 601e70f709..55e28e225a 100644 --- a/src/script/ismine.h +++ b/src/script/ismine.h @@ -9,6 +9,7 @@ #include <script/standard.h> #include <stdint.h> +#include <bitset> class CKeyStore; class CScript; @@ -16,10 +17,11 @@ class CScript; /** IsMine() return codes */ enum isminetype { - ISMINE_NO = 0, - ISMINE_WATCH_ONLY = 1, - ISMINE_SPENDABLE = 2, - ISMINE_ALL = ISMINE_WATCH_ONLY | ISMINE_SPENDABLE + ISMINE_NO = 0, + ISMINE_WATCH_ONLY = 1 << 0, + ISMINE_SPENDABLE = 1 << 1, + ISMINE_ALL = ISMINE_WATCH_ONLY | ISMINE_SPENDABLE, + ISMINE_ENUM_ELEMENTS, }; /** used for bitflags of isminetype */ typedef uint8_t isminefilter; @@ -27,4 +29,23 @@ typedef uint8_t isminefilter; isminetype IsMine(const CKeyStore& keystore, const CScript& scriptPubKey); isminetype IsMine(const CKeyStore& keystore, const CTxDestination& dest); +/** + * Cachable amount subdivided into watchonly and spendable parts. + */ +struct CachableAmount +{ + // NO and ALL are never (supposed to be) cached + std::bitset<ISMINE_ENUM_ELEMENTS> m_cached; + CAmount m_value[ISMINE_ENUM_ELEMENTS]; + inline void Reset() + { + m_cached.reset(); + } + void Set(isminefilter filter, CAmount value) + { + m_cached.set(filter); + m_value[filter] = value; + } +}; + #endif // BITCOIN_SCRIPT_ISMINE_H diff --git a/src/script/script.h b/src/script/script.h index 1d8ddba2f2..11e8661a5b 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -581,13 +581,4 @@ struct CScriptWitness std::string ToString() const; }; -class CReserveScript -{ -public: - CScript reserveScript; - virtual void KeepScript() {} - CReserveScript() {} - virtual ~CReserveScript() {} -}; - #endif // BITCOIN_SCRIPT_SCRIPT_H diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 320956d0c4..36dd68a3d8 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -483,7 +483,13 @@ bool HidingSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& inf bool FlatSigningProvider::GetCScript(const CScriptID& scriptid, CScript& script) const { return LookupHelper(scripts, scriptid, script); } bool FlatSigningProvider::GetPubKey(const CKeyID& keyid, CPubKey& pubkey) const { return LookupHelper(pubkeys, keyid, pubkey); } -bool FlatSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const { return LookupHelper(origins, keyid, info); } +bool FlatSigningProvider::GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const +{ + std::pair<CPubKey, KeyOriginInfo> out; + bool ret = LookupHelper(origins, keyid, out); + if (ret) info = std::move(out.second); + return ret; +} bool FlatSigningProvider::GetKey(const CKeyID& keyid, CKey& key) const { return LookupHelper(keys, keyid, key); } FlatSigningProvider Merge(const FlatSigningProvider& a, const FlatSigningProvider& b) diff --git a/src/script/sign.h b/src/script/sign.h index 491fb54c45..f746325b90 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -77,7 +77,7 @@ struct FlatSigningProvider final : public SigningProvider { std::map<CScriptID, CScript> scripts; std::map<CKeyID, CPubKey> pubkeys; - std::map<CKeyID, KeyOriginInfo> origins; + std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> origins; std::map<CKeyID, CKey> keys; bool GetCScript(const CScriptID& scriptid, CScript& script) const override; diff --git a/src/script/standard.h b/src/script/standard.h index fc20fb6a08..f16068c413 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -153,8 +153,7 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) * multisig scripts, this populates the addressRet vector with the pubkey IDs * and nRequiredRet with the n required to spend. For other destinations, * addressRet is populated with a single value and nRequiredRet is set to 1. - * Returns true if successful. Currently does not extract address from - * pay-to-witness scripts. + * Returns true if successful. * * Note: this function confuses destinations (a subset of CScripts that are * encodable as an address) with key identifiers (of keys involved in a diff --git a/src/serialize.h b/src/serialize.h index 2d0cfbbbf0..b001ee1324 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -89,6 +89,11 @@ template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj) obj = htole32(obj); s.write((char*)&obj, 4); } +template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); @@ -118,6 +123,12 @@ template<typename Stream> inline uint32_t ser_readdata32(Stream &s) s.read((char*)&obj, 4); return le32toh(obj); } +template<typename Stream> inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} template<typename Stream> inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; diff --git a/src/sync.cpp b/src/sync.cpp index 23ca866e53..e7c0a6f9bc 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -3,9 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <sync.h> +#include <tinyformat.h> #include <logging.h> #include <util/strencodings.h> +#include <util/threadnames.h> #include <stdio.h> @@ -37,23 +39,30 @@ void PrintLockContention(const char* pszName, const char* pszFile, int nLine) // struct CLockLocation { - CLockLocation(const char* pszName, const char* pszFile, int nLine, bool fTryIn) - { - mutexName = pszName; - sourceFile = pszFile; - sourceLine = nLine; - fTry = fTryIn; - } + CLockLocation( + const char* pszName, + const char* pszFile, + int nLine, + bool fTryIn, + const std::string& thread_name) + : fTry(fTryIn), + mutexName(pszName), + sourceFile(pszFile), + m_thread_name(thread_name), + sourceLine(nLine) {} std::string ToString() const { - return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); + return tfm::format( + "%s %s:%s%s (in thread %s)", + mutexName, sourceFile, itostr(sourceLine), (fTry ? " (TRY)" : ""), m_thread_name); } private: bool fTry; std::string mutexName; std::string sourceFile; + const std::string& m_thread_name; int sourceLine; }; @@ -125,7 +134,7 @@ static void push_lock(void* c, const CLockLocation& locklocation) std::pair<void*, void*> p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) continue; - lockdata.lockorders[p1] = g_lockstack; + lockdata.lockorders.emplace(p1, g_lockstack); std::pair<void*, void*> p2 = std::make_pair(c, i.first); lockdata.invlockorders.insert(p2); @@ -141,7 +150,7 @@ static void pop_lock() void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) { - push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry)); + push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry, util::ThreadGetInternalName())); } void LeaveCritical() diff --git a/src/sync.h b/src/sync.h index 3857eda56b..2667fb52f9 100644 --- a/src/sync.h +++ b/src/sync.h @@ -198,6 +198,16 @@ using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove LeaveCritical(); \ } +//! Run code while locking a mutex. +//! +//! Examples: +//! +//! WITH_LOCK(cs, shared_val = shared_val + 1); +//! +//! int val = WITH_LOCK(cs, return shared_val); +//! +#define WITH_LOCK(cs, code) [&] { LOCK(cs); code; }() + class CSemaphore { private: diff --git a/src/test/README.md b/src/test/README.md index f2a4cb1818..0017e3de26 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -42,7 +42,7 @@ unit tests as possible). The build system is setup to compile an executable called `test_bitcoin` that runs all of the unit tests. The main source file is called -test_bitcoin.cpp. To add a new unit test file to our test suite you need +setup_common.cpp. To add a new unit test file to our test suite you need to add the file to `src/Makefile.test.include`. The pattern is to create one test file for each class or source file for which you want to create unit tests. The file naming convention is `<source_filename>_tests.cpp` diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 22347fbc57..eeb54b4cf0 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 9eded4f5b2..f255691704 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/system.h> #include <support/allocators/secure.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <memory> diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 1ff040b077..378fe285d5 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <amount.h> #include <policy/feerate.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 77b6008fd0..809c627d27 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <arith_uint256.h> #include <string> #include <version.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 32af843bf6..b3bed2434c 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index f8f9b3c1a7..cb376cddb6 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/base58_encode_decode.json.h> #include <base58.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/strencodings.h> #include <univalue.h> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 0abbb682a7..9ffffb0b7d 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index 6ecc9ac705..0ba492c24e 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bech32.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index c9951f4b7e..0c0423c0db 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index b61152985f..13afcca375 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -3,7 +3,7 @@ #include <stdlib.h> #include <rpc/blockchain.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 607af8a32a..f57e1a0ebd 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #include <pow.h> #include <random.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> @@ -386,6 +386,7 @@ BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationOverflowTest) { BOOST_CHECK(0); } catch(std::ios_base::failure &) { // deserialize should fail + BOOST_CHECK(true); // Needed to suppress "Test case [...] did not check any assertions" } } diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp new file mode 100644 index 0000000000..db0b973463 --- /dev/null +++ b/src/test/blockfilter_index_tests.cpp @@ -0,0 +1,307 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <blockfilter.h> +#include <chainparams.h> +#include <consensus/validation.h> +#include <index/blockfilterindex.h> +#include <miner.h> +#include <pow.h> +#include <test/setup_common.h> +#include <script/standard.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) + +static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, + BlockFilter& filter) +{ + CBlock block; + if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + return false; + } + + CBlockUndo block_undo; + if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + return false; + } + + filter = BlockFilter(filter_type, block, block_undo); + return true; +} + +static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, + uint256& last_header) +{ + BlockFilter expected_filter; + if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) { + BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight); + return false; + } + + BlockFilter filter; + uint256 filter_header; + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + BOOST_CHECK(filter_index.LookupFilter(block_index, filter)); + BOOST_CHECK(filter_index.LookupFilterHeader(block_index, filter_header)); + BOOST_CHECK(filter_index.LookupFilterRange(block_index->nHeight, block_index, filters)); + BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index, + filter_hashes)); + + BOOST_CHECK_EQUAL(filters.size(), 1); + BOOST_CHECK_EQUAL(filter_hashes.size(), 1); + + BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash()); + BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header)); + BOOST_CHECK_EQUAL(filters[0].GetHash(), expected_filter.GetHash()); + BOOST_CHECK_EQUAL(filter_hashes[0], expected_filter.GetHash()); + + filters.clear(); + filter_hashes.clear(); + last_header = filter_header; + return true; +} + +static CBlock CreateBlock(const CBlockIndex* prev, + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey) +{ + const CChainParams& chainparams = Params(); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + CBlock& block = pblocktemplate->block; + block.hashPrevBlock = prev->GetBlockHash(); + block.nTime = prev->nTime + 1; + + // Replace mempool-selected txns with just coinbase plus passed-in txns: + block.vtx.resize(1); + for (const CMutableTransaction& tx : txns) { + block.vtx.push_back(MakeTransactionRef(tx)); + } + // IncrementExtraNonce creates a valid coinbase and merkleRoot + unsigned int extraNonce = 0; + IncrementExtraNonce(&block, prev, extraNonce); + + while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; + + return block; +} + +static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, + size_t length, std::vector<std::shared_ptr<CBlock>>& chain) +{ + std::vector<CMutableTransaction> no_txns; + + chain.resize(length); + for (auto& block : chain) { + block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key)); + CBlockHeader header = block->GetBlockHeader(); + + CValidationState state; + if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) { + return false; + } + } + + return true; +} + +BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup) +{ + BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); + + uint256 last_header; + + // Filter should not be found in the index before it is started. + { + LOCK(cs_main); + + BlockFilter filter; + uint256 filter_header; + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + for (const CBlockIndex* block_index = chainActive.Genesis(); + block_index != nullptr; + block_index = chainActive.Next(block_index)) { + BOOST_CHECK(!filter_index.LookupFilter(block_index, filter)); + BOOST_CHECK(!filter_index.LookupFilterHeader(block_index, filter_header)); + BOOST_CHECK(!filter_index.LookupFilterRange(block_index->nHeight, block_index, filters)); + BOOST_CHECK(!filter_index.LookupFilterHashRange(block_index->nHeight, block_index, + filter_hashes)); + } + } + + // BlockUntilSyncedToCurrentChain should return false before index is started. + BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain()); + + filter_index.Start(); + + // Allow filter index to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!filter_index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + // Check that filter index has all blocks that were in the chain before it started. + { + LOCK(cs_main); + const CBlockIndex* block_index; + for (block_index = chainActive.Genesis(); + block_index != nullptr; + block_index = chainActive.Next(block_index)) { + CheckFilterLookups(filter_index, block_index, last_header); + } + } + + // Create two forks. + const CBlockIndex* tip; + { + LOCK(cs_main); + tip = chainActive.Tip(); + } + CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID()); + std::vector<std::shared_ptr<CBlock>> chainA, chainB; + BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainA)); + BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainB)); + + // Check that new blocks on chain A get indexed. + uint256 chainA_last_header = last_header; + for (size_t i = 0; i < 2; i++) { + const auto& block = chainA[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + } + + // Reorg to chain B. + uint256 chainB_last_header = last_header; + for (size_t i = 0; i < 3; i++) { + const auto& block = chainB[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainB_last_header); + } + + // Check that filters for stale blocks on A can be retrieved. + chainA_last_header = last_header; + for (size_t i = 0; i < 2; i++) { + const auto& block = chainA[i]; + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + } + + // Reorg back to chain A. + for (size_t i = 2; i < 4; i++) { + const auto& block = chainA[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + } + + // Check that chain A and B blocks can be retrieved. + chainA_last_header = last_header; + chainB_last_header = last_header; + for (size_t i = 0; i < 3; i++) { + const CBlockIndex* block_index; + + { + LOCK(cs_main); + block_index = LookupBlockIndex(chainA[i]->GetHash()); + } + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + + { + LOCK(cs_main); + block_index = LookupBlockIndex(chainB[i]->GetHash()); + } + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainB_last_header); + } + + // Test lookups for a range of filters/hashes. + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + { + LOCK(cs_main); + tip = chainActive.Tip(); + } + BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters)); + BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes)); + + BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1); + BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1); + + filters.clear(); + filter_hashes.clear(); + + filter_index.Interrupt(); + filter_index.Stop(); +} + +BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) +{ + SetDataDir("tempdir"); + + BlockFilterIndex* filter_index; + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); + + BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index != nullptr); + BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC); + + // Initialize returns false if index already exists. + BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + int iter_count = 0; + ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; }); + BOOST_CHECK_EQUAL(iter_count, 1); + + BOOST_CHECK(DestroyBlockFilterIndex(BlockFilterType::BASIC)); + + // Destroy returns false because index was already destroyed. + BOOST_CHECK(!DestroyBlockFilterIndex(BlockFilterType::BASIC)); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); + + // Reinitialize index. + BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + DestroyAllBlockFilterIndexes(); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 625362f446..df0a041e0e 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/blockfilters.json.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <blockfilter.h> #include <core_io.h> @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor) BOOST_AUTO_TEST_CASE(blockfilter_basic_test) { - CScript included_scripts[5], excluded_scripts[3]; + CScript included_scripts[5], excluded_scripts[4]; // First two are outputs on a single transaction. included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG; @@ -73,14 +73,19 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) // This script is not related to the block at all. excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG; + // OP_RETURN is non-standard since it's not followed by a data push, but is still excluded from + // filter. + excluded_scripts[2] << OP_RETURN << OP_4 << OP_ADD << OP_8 << OP_EQUAL; + CMutableTransaction tx_1; tx_1.vout.emplace_back(100, included_scripts[0]); tx_1.vout.emplace_back(200, included_scripts[1]); + tx_1.vout.emplace_back(0, excluded_scripts[0]); CMutableTransaction tx_2; tx_2.vout.emplace_back(300, included_scripts[2]); - tx_2.vout.emplace_back(0, excluded_scripts[0]); - tx_2.vout.emplace_back(400, excluded_scripts[2]); // Script is empty + tx_2.vout.emplace_back(0, excluded_scripts[2]); + tx_2.vout.emplace_back(400, excluded_scripts[3]); // Script is empty CBlock block; block.vtx.push_back(MakeTransactionRef(tx_1)); @@ -90,7 +95,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) block_undo.vtxundo.emplace_back(); block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[3]), 1000, true); block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(600, included_scripts[4]), 10000, false); - block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[2]), 100000, false); + block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[3]), 100000, false); BlockFilter block_filter(BlockFilterType::BASIC, block, block_undo); const GCSFilter& filter = block_filter.GetFilter(); @@ -112,6 +117,12 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) BOOST_CHECK_EQUAL(block_filter.GetFilterType(), block_filter2.GetFilterType()); BOOST_CHECK_EQUAL(block_filter.GetBlockHash(), block_filter2.GetBlockHash()); BOOST_CHECK(block_filter.GetEncodedFilter() == block_filter2.GetEncodedFilter()); + + BlockFilter default_ctor_block_filter_1; + BlockFilter default_ctor_block_filter_2; + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetFilterType(), default_ctor_block_filter_2.GetFilterType()); + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetBlockHash(), default_ctor_block_filter_2.GetBlockHash()); + BOOST_CHECK(default_ctor_block_filter_1.GetEncodedFilter() == default_ctor_block_filter_2.GetEncodedFilter()); } BOOST_AUTO_TEST_CASE(blockfilters_json_test) @@ -168,4 +179,16 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test) } } +BOOST_AUTO_TEST_CASE(blockfilter_type_names) +{ + BOOST_CHECK_EQUAL(BlockFilterTypeName(BlockFilterType::BASIC), "basic"); + BOOST_CHECK_EQUAL(BlockFilterTypeName(static_cast<BlockFilterType>(255)), ""); + + BlockFilterType filter_type; + BOOST_CHECK(BlockFilterTypeByName("basic", filter_type)); + BOOST_CHECK_EQUAL(filter_type, BlockFilterType::BASIC); + + BOOST_CHECK(!BlockFilterTypeByName("unknown", filter_type)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index f58dd20efc..4421494007 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 8572926193..8fd4e5d5d6 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compat/byteswap.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 3469c6dfba..408a7fbda4 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <util/time.h> #include <validation.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> @@ -167,7 +167,6 @@ static void Correct_Queue_range(std::vector<size_t> range) BOOST_REQUIRE(control.Wait()); if (FakeCheckCheckCompletion::n_calls != i) { BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); - BOOST_TEST_MESSAGE("Failure on trial " << i << " expected, got " << FakeCheckCheckCompletion::n_calls); } } tg.interrupt_all(); diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index f6b97a6868..232c077c68 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <coins.h> #include <consensus/validation.h> #include <script/standard.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index e686c05165..6cef8cd8a8 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 86cb00a78f..0d05b6514f 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/aes.h> #include <crypto/chacha20.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -12,13 +13,11 @@ #include <crypto/hmac_sha512.h> #include <random.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> #include <boost/test/unit_test.hpp> -#include <openssl/aes.h> -#include <openssl/evp.h> BOOST_FIXTURE_TEST_SUITE(crypto_tests, BasicTestingSetup) @@ -66,26 +65,6 @@ static void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, TestVector(CHMAC_SHA512(key.data(), key.size()), ParseHex(hexin), ParseHex(hexout)); } -static void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> buf, buf2; - - assert(key.size() == 16); - assert(in.size() == 16); - assert(correctout.size() == 16); - AES128Encrypt enc(key.data()); - buf.resize(correctout.size()); - buf2.resize(correctout.size()); - enc.Encrypt(buf.data(), in.data()); - BOOST_CHECK_EQUAL(HexStr(buf), HexStr(correctout)); - AES128Decrypt dec(key.data()); - dec.Decrypt(buf2.data(), buf.data()); - BOOST_CHECK_EQUAL(HexStr(buf2), HexStr(in)); -} - static void TestAES256(const std::string &hexkey, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -105,47 +84,6 @@ static void TestAES256(const std::string &hexkey, const std::string &hexin, cons BOOST_CHECK(buf == in); } -static void TestAES128CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> iv = ParseHex(hexiv); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> realout(in.size() + AES_BLOCKSIZE); - - // Encrypt the plaintext and verify that it equals the cipher - AES128CBCEncrypt enc(key.data(), iv.data(), pad); - int size = enc.Encrypt(in.data(), in.size(), realout.data()); - realout.resize(size); - BOOST_CHECK(realout.size() == correctout.size()); - BOOST_CHECK_MESSAGE(realout == correctout, HexStr(realout) + std::string(" != ") + hexout); - - // Decrypt the cipher and verify that it equals the plaintext - std::vector<unsigned char> decrypted(correctout.size()); - AES128CBCDecrypt dec(key.data(), iv.data(), pad); - size = dec.Decrypt(correctout.data(), correctout.size(), decrypted.data()); - decrypted.resize(size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(decrypted == in, HexStr(decrypted) + std::string(" != ") + hexin); - - // Encrypt and re-decrypt substrings of the plaintext and verify that they equal each-other - for(std::vector<unsigned char>::iterator i(in.begin()); i != in.end(); ++i) - { - std::vector<unsigned char> sub(i, in.end()); - std::vector<unsigned char> subout(sub.size() + AES_BLOCKSIZE); - int _size = enc.Encrypt(sub.data(), sub.size(), subout.data()); - if (_size != 0) - { - subout.resize(_size); - std::vector<unsigned char> subdecrypted(subout.size()); - _size = dec.Decrypt(subout.data(), subout.size(), subdecrypted.data()); - subdecrypted.resize(_size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); - } - } -} - static void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -200,6 +138,17 @@ static void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t see BOOST_CHECK(out == outres); } +static void TestPoly1305(const std::string &hexmessage, const std::string &hexkey, const std::string& hextag) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + std::vector<unsigned char> m = ParseHex(hexmessage); + std::vector<unsigned char> tag = ParseHex(hextag); + std::vector<unsigned char> tagres; + tagres.resize(POLY1305_TAGLEN); + poly1305_auth(tagres.data(), m.data(), m.size(), key.data()); + BOOST_CHECK(tag == tagres); +} + static std::string LongTestString() { std::string ret; for (int i=0; i<200000; i++) { @@ -428,14 +377,9 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { BOOST_AUTO_TEST_CASE(aes_testvectors) { // AES test vectors from FIPS 197. - TestAES128("000102030405060708090a0b0c0d0e0f", "00112233445566778899aabbccddeeff", "69c4e0d86a7b0430d8cdb78070b4c55a"); TestAES256("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "00112233445566778899aabbccddeeff", "8ea2b7ca516745bfeafc49904b496089"); // AES-ECB test vectors from NIST sp800-38a. - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "6bc1bee22e409f96e93d7e117393172a", "3ad77bb40d7a3660a89ecaf32466ef97"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "ae2d8a571e03ac9c9eb76fac45af8e51", "f5d3d58503b9699de785895a96fdbaaf"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "30c81c46a35ce411e5fbc1191a0a52ef", "43b1cd7f598ece23881b00e3ed030688"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "f69f2445df4f9b17ad2b417be66c3710", "7b0c785e27e8ad3f8223207104725dd4"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "6bc1bee22e409f96e93d7e117393172a", "f3eed1bdb5d2a03c064b5a7e3db181f8"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "ae2d8a571e03ac9c9eb76fac45af8e51", "591ccb10d410ed26dc5ba74a31362870"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "30c81c46a35ce411e5fbc1191a0a52ef", "b6ed21b99ca6f4f9f153e7b1beafed1d"); @@ -443,27 +387,6 @@ BOOST_AUTO_TEST_CASE(aes_testvectors) { } BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { - - // NIST AES CBC 128-bit encryption test-vectors - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", false, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", false, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", false, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", false, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a7"); - - // The same vectors with padding enabled - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", true, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d8964e0b149c10b7b682e6e39aaeb731c"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", true, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b255e21d7100b988ffec32feeafaf23538"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", true, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516f6eccda327bf8e5ec43718b0039adceb"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", true, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a78cb82807230e1321d3fae00d18cc2012"); - // NIST AES CBC 256-bit encryption test-vectors TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ "000102030405060708090A0B0C0D0E0F", false, "6bc1bee22e409f96e93d7e117393172a", \ @@ -524,6 +447,76 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) "fab78c9"); } +BOOST_AUTO_TEST_CASE(poly1305_testvector) +{ + // RFC 7539, section 2.5.2. + TestPoly1305("43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"); + + // RFC 7539, section A.3. + TestPoly1305("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"); + + TestPoly1305("2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"); + + TestPoly1305("ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"); + + TestPoly1305("02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"); +} + BOOST_AUTO_TEST_CASE(countbits_tests) { FastRandomContext ctx; diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index d8286520ec..d38ede691a 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include <cuckoocache.h> #include <script/sigcache.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <random.h> #include <thread> diff --git a/src/test/data/blockfilters.json b/src/test/data/blockfilters.json index 134b788eed..8945296a07 100644 --- a/src/test/data/blockfilters.json +++ b/src/test/data/blockfilters.json @@ -3,9 +3,11 @@ [0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"], [2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""], [3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""], +[15007,"0000000038c44c703bae0f98cdd6bf30922326340a5996cc692aaae8bacf47ad","0100000002394092aa378fe35d7e9ac79c869b975c4de4374cd75eb5484b0e1e00000000eb9b8670abd44ad6c55cee18e3020fb0c6519e7004b01a16e9164867531b67afc33bc94fffff001d123f10050101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e04c33bc94f0115062f503253482fffffffff0100f2052a01000000232103f268e9ae07e0f8cb2f6e901d87c510d650b97230c0365b021df8f467363cafb1ac00000000",[],"18b5c2b0146d2d09d24fb00ff5b52bd0742f36c9e65527abdb9de30c027a4748","013c3710","07384b01311867949e0c046607c66b7a766d338474bb67f66c8ae9dbd454b20e","Tx has non-standard OP_RETURN output followed by opcodes"], [49291,"0000000018b07dca1b28b4b5a119f6d6e71698ce1ed96f143f54179ce177a19c","02000000abfaf47274223ca2fea22797e44498240e482cb4c2f2baea088962f800000000604b5b52c32305b15d7542071d8b04e750a547500005d4010727694b6e72a776e55d0d51ffff001d211806480201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d038bc0000102062f503253482fffffffff01a078072a01000000232102971dd6034ed0cf52450b608d196c07d6345184fcb14deb277a6b82d526a6163dac0000000001000000081cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff000000009300493046022100866859c21f306538152e83f115bcfbf59ab4bb34887a88c03483a5dff9895f96022100a6dfd83caa609bf0516debc2bf65c3df91813a4842650a1858b3f61cfa8af249014730440220296d4b818bb037d0f83f9f7111665f49532dfdcbec1e6b784526e9ac4046eaa602204acf3a5cb2695e8404d80bf49ab04828bcbe6fc31d25a2844ced7a8d24afbdff01ffffffff1cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff020000009400483045022100e87899175991aa008176cb553c6f2badbb5b741f328c9845fcab89f8b18cae2302200acce689896dc82933015e7230e5230d5cff8a1ffe82d334d60162ac2c5b0c9601493046022100994ad29d1e7b03e41731a4316e5f4992f0d9b6e2efc40a1ccd2c949b461175c502210099b69fdc2db00fbba214f16e286f6a49e2d8a0d5ffc6409d87796add475478d601ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272010000009500493046022100a27400ba52fd842ce07398a1de102f710a10c5599545e6c95798934352c2e4df022100f6383b0b14c9f64b6718139f55b6b9494374755b86bae7d63f5d3e583b57255a01493046022100fdf543292f34e1eeb1703b264965339ec4a450ec47585009c606b3edbc5b617b022100a5fbb1c8de8aaaa582988cdb23622838e38de90bebcaab3928d949aa502a65d401ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272020000009400493046022100ac626ac3051f875145b4fe4cfe089ea895aac73f65ab837b1ac30f5d875874fa022100bc03e79fa4b7eb707fb735b95ff6613ca33adeaf3a0607cdcead4cfd3b51729801483045022100b720b04a5c5e2f61b7df0fcf334ab6fea167b7aaede5695d3f7c6973496adbf1022043328c4cc1cdc3e5db7bb895ccc37133e960b2fd3ece98350f774596badb387201ffffffff23a8733e349c97d6cd90f520fdd084ba15ce0a395aad03cd51370602bb9e5db3010000004a00483045022100e8556b72c5e9c0da7371913a45861a61c5df434dfd962de7b23848e1a28c86ca02205d41ceda00136267281be0974be132ac4cda1459fe2090ce455619d8b91045e901ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a6000000006a473044022040a1c631554b8b210fbdf2a73f191b2851afb51d5171fb53502a3a040a38d2c0022040d11cf6e7b41fe1b66c3d08f6ada1aee07a047cb77f242b8ecc63812c832c9a012102bcfad931b502761e452962a5976c79158a0f6d307ad31b739611dac6a297c256ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a601000000930048304502205b109df098f7e932fbf71a45869c3f80323974a826ee2770789eae178a21bfc8022100c0e75615e53ee4b6e32b9bb5faa36ac539e9c05fa2ae6b6de5d09c08455c8b9601483045022009fb7d27375c47bea23b24818634df6a54ecf72d52e0c1268fb2a2c84f1885de022100e0ed4f15d62e7f537da0d0f1863498f9c7c0c0a4e00e4679588c8d1a9eb20bb801ffffffffa563c3722b7b39481836d5edfc1461f97335d5d1e9a23ade13680d0e2c1c371f030000006c493046022100ecc38ae2b1565643dc3c0dad5e961a5f0ea09cab28d024f92fa05c922924157e022100ebc166edf6fbe4004c72bfe8cf40130263f98ddff728c8e67b113dbd621906a601210211a4ed241174708c07206601b44a4c1c29e5ad8b1f731c50ca7e1d4b2a06dc1fffffffff02d0223a00000000001976a91445db0b779c0b9fa207f12a8218c94fc77aff504588ac80f0fa02000000000000000000",["5221033423007d8f263819a2e42becaaf5b06f34cb09919e06304349d950668209eaed21021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","522102a7ae1e0971fc1689bd66d2a7296da3a1662fd21a53c9e38979e0f090a375c12d21022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","512103b9d1d0e2b4355ec3cdef7c11a5c0beff9e8b8d8372ab4b4e0aaf30e80173001951ae","76a9149144761ebaccd5b4bbdc2a35453585b5637b2f8588ac","522103f1848b40621c5d48471d9784c8174ca060555891ace6d2b03c58eece946b1a9121020ee5d32b54d429c152fdc7b1db84f2074b0564d35400d89d11870f9273ec140c52ae","76a914f4fa1cc7de742d135ea82c17adf0bb9cf5f4fb8388ac"],"ed47705334f4643892ca46396eb3f4196a5e30880589e4009ef38eae895d4a13","0afbc2920af1b027f31f87b592276eb4c32094bb4d3697021b4c6380","b6d98692cec5145f67585f3434ec3c2b3030182e1cb3ec58b855c5c164dfaaa3","Tx pays to empty output script"], -[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"b109139671dbedc2b6fcd499a5480a7461ae458af8ff9411d819aa64ba6995d1","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","a0af77e0a7ed20ea78d2def3200cc24f08217dcd51755c7c7feb0e2ba8316c2d","Tx spends from empty output script"], -[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"da49977ba1ee0d620a2c4f8f646b03cd0d230f5c6c994722e3ba884889f0be1a","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","4cd9dd007a325199102f1fc0b7d77ca25ee3c84d46018c4353ecfcb56c0d3e7a","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"], -[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"e9d729b72d533c29abe5276d5cf6c152f3723f10efe000b1e0c9ca5265a8beb6","010c0b40","e6137ae5a8424c40da1e5023c16975cc97b09300b4c050e6b1c713add3836c40","Coinbase tx has unparseable output script"], -[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"a4a4d6c6034da8aa06f01fe71f1fffbd79e032006b07f6c7a2c60a66aa310c01","0385acb4f0fe889ef0","3588f34fbbc11640f9ed40b2a66a4e096215d50389691309c1dac74d4268aa81","Includes witness data"] +[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"d34ef98386f413769502808d4bac5f20f8dfd5bffc9eedafaa71de0eb1f01489","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","c582d51c0ca365e3fcf36c51cb646d7f83a67e867cb4743fd2128e3e022b700c","Tx spends from empty output script"], +[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"8f13b9a9c85611635b47906c3053ac53cfcec7211455d4cb0d63dc9acc13d472","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","546c574a0472144bcaf9b6aeabf26372ad87c7af7d1ee0dbfae5e099abeae49c","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"], +[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"fe4d230dbb0f4fec9bed23a5283e08baf996e3f32b93f52c7de1f641ddfd04ad","010c0b40","0965a544743bbfa36f254446e75630c09404b3d164a261892372977538928ed5","Coinbase tx has unparseable output script"], +[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"31d66d516a9eda7de865df29f6ef6cb8e4bf9309e5dac899968a9a62a5df61e3","0385acb4f0fe889ef0","4e6d564c2a2452065c205dd7eb2791124e0c4e0dbb064c410c24968572589dec","Includes witness data"], +[1414221,"0000000000000027b2b3b3381f114f674f481544ff2be37ae3788d7e078383b1","000000204ea88307a7959d8207968f152bedca5a93aefab253f1fb2cfb032a400000000070cebb14ec6dbc27a9dfd066d9849a4d3bac5f674665f73a5fe1de01a022a0c851fda85bf05f4c19a779d1450102000000010000000000000000000000000000000000000000000000000000000000000000ffffffff18034d94154d696e6572476174653030310d000000f238f401ffffffff01c817a804000000000000000000",[],"5e5e12d90693c8e936f01847859404c67482439681928353ca1296982042864e","00","021e8882ef5a0ed932edeebbecfeda1d7ce528ec7b3daa27641acf1189d7b5dc","Empty data"] ] diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 94e8c95345..0bde92c18d 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <dbwrapper.h> #include <uint256.h> #include <random.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <memory> diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index e5d62a3ab2..bcb9a7c181 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ #include <util/system.h> #include <validation.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <stdint.h> diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index ff2b8d4fc9..f5bda7d5e6 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <string> #include <script/sign.h> #include <script/standard.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> #include <script/descriptor.h> #include <util/strencodings.h> @@ -154,8 +154,8 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: // Test whether the observed key path is present in the 'paths' variable (which contains expected, unobserved paths), // and then remove it from that set. for (const auto& origin : script_provider.origins) { - BOOST_CHECK_MESSAGE(paths.count(origin.second.path), "Unexpected key path: " + prv); - left_paths.erase(origin.second.path); + BOOST_CHECK_MESSAGE(paths.count(origin.second.second.path), "Unexpected key path: " + prv); + left_paths.erase(origin.second.second.path); } } } diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 index 0000000000..1db2f8054c --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <flatfile.h> +#include <test/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) +{ + auto data_dir = SetDataDir("flatfile_test"); + + FlatFilePos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1("A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but the main benefits are " + "lost if a trusted third party is still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(FlatFilePos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(FlatFilePos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(FlatFilePos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index 93aee10bb7..6bd6bb1be3 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <fs.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 859fba0bdc..97d7633715 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index ad62a5faf0..8b03a7e46e 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -10,8 +10,6 @@ #include <vector> -const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; - void test_one_input(std::vector<uint8_t> buffer); #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 8048238028..8a42344642 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <vector> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index e8e5040855..325b7002f2 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/siphash.h> #include <hash.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index bf295042de..e924f27d1b 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <key_io.h> #include <script/script.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_properties.cpp b/src/test/key_properties.cpp index c564b4eab8..8b508ed7f7 100644 --- a/src/test/key_properties.cpp +++ b/src/test/key_properties.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <vector> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index a768b4bcbd..e816546e62 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <vector> diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index 0788f75b04..00b36f51fb 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <limitedmap.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/main.cpp b/src/test/main.cpp new file mode 100644 index 0000000000..ff3f36b561 --- /dev/null +++ b/src/test/main.cpp @@ -0,0 +1,7 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#define BOOST_TEST_MODULE Bitcoin Core Test Suite + +#include <boost/test/unit_test.hpp> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 23ca9d89ae..0f74b379c0 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <txmempool.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> #include <list> diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 4cdf0f003e..1684258c9f 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/merkle.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 4978593285..eac43471c7 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> #include <uint256.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 5ba1df2ec2..6ed4359059 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <memory> diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 8afe4b8a59..682f1bee26 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <script/sign.h> #include <script/ismine.h> #include <uint256.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b4ae8e9765..54d18c0a1c 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> #include <hash.h> diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 0d557cff13..dd5e3eb6d5 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netbase.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/strencodings.h> #include <string> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index 5020192804..c5513ae9fa 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <uint256.h> #include <arith_uint256.h> #include <version.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 7b274a1658..149094fc00 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 26cdc9bc5c..653433bfce 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,7 +7,7 @@ #include <pow.h> #include <random.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 7341389208..141be50f50 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <serialize.h> #include <streams.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index bdb411d53f..2b01acf7fa 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <support/events.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 8194070aba..e6fbe2355d 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <random.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 91c76fda88..69db9dcf4e 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <reverselock.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index ff48398925..07d1326bcb 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <key_io.h> #include <netbase.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> @@ -31,10 +31,9 @@ UniValue CallRPC(std::string args) request.strMethod = strMethod; request.params = RPCConvertValues(strMethod, vArgs); request.fHelp = false; - BOOST_CHECK(tableRPC[strMethod]); - rpcfn_type method = tableRPC[strMethod]->actor; + if (RPCIsInWarmup(nullptr)) SetRPCWarmupFinished(); try { - UniValue result = (*method)(request); + UniValue result = tableRPC.execute(request); return result; } catch (const UniValue& objError) { diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 8085a21928..891aa8e5c3 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compat/sanity.h> #include <key.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 100d65b779..42242b962b 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <random.h> #include <scheduler.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/thread.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index 3a2a11ef98..0ce5f09e42 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,9 +10,10 @@ #include <policy/policy.h> #include <script/script.h> #include <script/script_error.h> +#include <policy/settings.h> #include <script/sign.h> #include <script/ismine.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index bde82018c7..36a2b1bee5 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #include <script/script.h> #include <script/script_error.h> #include <script/standard.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 87c3e74df0..588ae55a69 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,8 +12,8 @@ #include <script/sign.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/test_bitcoin.h> -#include <rpc/server.h> +#include <test/setup_common.h> +#include <rpc/util.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index f9b407ce3e..e7916f5000 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/scriptnum10.h> #include <script/script.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> #include <limits.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 002f61f6a2..2fab309aa4 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <serialize.h> #include <streams.h> #include <hash.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <stdint.h> diff --git a/src/test/test_bitcoin.cpp b/src/test/setup_common.cpp index 0c3fb7c398..6b671b2400 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/setup_common.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <banman.h> #include <chainparams.h> @@ -19,6 +19,7 @@ #include <script/sigcache.h> #include <streams.h> #include <ui_interface.h> +#include <util/validation.h> #include <validation.h> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; @@ -32,7 +33,7 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) } BasicTestingSetup::BasicTestingSetup(const std::string& chainName) - : m_path_root(fs::temp_directory_path() / "test_bitcoin" / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) + : m_path_root(fs::temp_directory_path() / "test_common_" PACKAGE_NAME / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) { SHA256AutoDetect(); ECC_Start(); @@ -41,11 +42,12 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName) InitSignatureCache(); InitScriptExecutionCache(); fCheckBlockIndex = true; - // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. - // TODO: fix the code to support SegWit blocks. - gArgs.ForceSetArg("-vbparams", strprintf("segwit:0:%d", (int64_t)Consensus::BIP9Deployment::NO_TIMEOUT)); SelectParams(chainName); - noui_connect(); + static bool noui_connected = false; + if (!noui_connected) { + noui_connect(); + noui_connected = true; + } } BasicTestingSetup::~BasicTestingSetup() @@ -66,36 +68,36 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha { SetDataDir("tempdir"); const CChainParams& chainparams = Params(); - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - - RegisterAllCoreRPCCommands(tableRPC); - ClearDatadirCache(); - - // We have to run a scheduler thread to prevent ActivateBestChain - // from blocking due to queue overrun. - threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); - GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - - mempool.setSanityCheck(1.0); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); - pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - if (!LoadGenesisBlock(chainparams)) { - throw std::runtime_error("LoadGenesisBlock failed."); - } - { - CValidationState state; - if (!ActivateBestChain(state, chainparams)) { - throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); - } - } - nScriptCheckThreads = 3; - for (int i=0; i < nScriptCheckThreads-1; i++) - threadGroup.create_thread(&ThreadScriptCheck); - - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + + RegisterAllCoreRPCCommands(tableRPC); + ClearDatadirCache(); + + // We have to run a scheduler thread to prevent ActivateBestChain + // from blocking due to queue overrun. + threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + + mempool.setSanityCheck(1.0); + pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); + if (!LoadGenesisBlock(chainparams)) { + throw std::runtime_error("LoadGenesisBlock failed."); + } + + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); + } + + nScriptCheckThreads = 3; + for (int i = 0; i < nScriptCheckThreads - 1; i++) + threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); + + g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. } TestingSetup::~TestingSetup() @@ -114,6 +116,11 @@ TestingSetup::~TestingSetup() TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) { + // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. + // TODO: fix the code to support SegWit blocks. + gArgs.ForceSetArg("-vbparams", strprintf("segwit:0:%d", (int64_t)Consensus::BIP9Deployment::NO_TIMEOUT)); + SelectParams(CBaseChainParams::REGTEST); + // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; diff --git a/src/test/test_bitcoin.h b/src/test/setup_common.h index 4a06845683..893eca216d 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/setup_common.h @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_TEST_BITCOIN_H -#define BITCOIN_TEST_TEST_BITCOIN_H +#ifndef BITCOIN_TEST_SETUP_COMMON_H +#define BITCOIN_TEST_SETUP_COMMON_H #include <chainparamsbase.h> #include <fs.h> @@ -71,10 +71,6 @@ private: /** Testing setup that configures a complete environment. * Included are data directory, coins database, script check threads setup. */ -class CConnman; -class CNode; - -class PeerLogicValidation; struct TestingSetup : public BasicTestingSetup { boost::thread_group threadGroup; CScheduler scheduler; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 04d5462acb..15f8db899b 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <consensus/tx_verify.h> +#include <consensus/tx_check.h> #include <consensus/validation.h> #include <test/data/sighash.json.h> #include <hash.h> @@ -10,7 +10,7 @@ #include <script/script.h> #include <serialize.h> #include <streams.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/system.h> #include <util/strencodings.h> #include <version.h> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 6a0349cd4e..4efa023fbb 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <script/script.h> #include <script/standard.h> #include <uint256.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 5c46976ace..3d39dfdb75 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> #include <util/system.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <vector> @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_test) // Pick a random element in vBlocksMain. int r = InsecureRandRange(vBlocksMain.size()); int64_t test_time = vBlocksMain[r].nTime; - CBlockIndex *ret = chain.FindEarliestAtLeast(test_time); + CBlockIndex* ret = chain.FindEarliestAtLeast(test_time, 0); BOOST_CHECK(ret->nTimeMax >= test_time); BOOST_CHECK((ret->pprev==nullptr) || ret->pprev->nTimeMax < test_time); BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret); @@ -158,22 +158,34 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test) CChain chain; chain.SetTip(&blocks.back()); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6); - BOOST_CHECK(!chain.FindEarliestAtLeast(350)); - - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0); - - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min())->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1)->nHeight, 0); - BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max())); - BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max())); - BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1)); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150, 0)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200, 0)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250, 0)->nHeight, 6); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300, 0)->nHeight, 6); + BOOST_CHECK(!chain.FindEarliestAtLeast(350, 0)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1, 0)->nHeight, 0); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min(), 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1, 0)->nHeight, 0); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max(), 0)); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max(), 0)); + BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1, 0)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, -1)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 3)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 8)->nHeight, 8); + BOOST_CHECK(!chain.FindEarliestAtLeast(0, 9)); + + CBlockIndex* ret1 = chain.FindEarliestAtLeast(100, 2); + BOOST_CHECK(ret1->nTimeMax >= 100 && ret1->nHeight == 2); + BOOST_CHECK(!chain.FindEarliestAtLeast(300, 9)); + CBlockIndex* ret2 = chain.FindEarliestAtLeast(200, 4); + BOOST_CHECK(ret2->nTimeMax >= 200 && ret2->nHeight == 4); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index a1940eb80e..4e37199c63 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <streams.h> #include <support/allocators/zeroafterfree.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index df0380546e..c1399d2dbe 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2017 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <sync.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp deleted file mode 100644 index 46b63b93b4..0000000000 --- a/src/test/test_bitcoin_main.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#define BOOST_TEST_MODULE Bitcoin Test Suite - -#include <banman.h> -#include <net.h> - -#include <memory> - -#include <boost/test/unit_test.hpp> - -std::unique_ptr<CConnman> g_connman; -std::unique_ptr<BanMan> g_banman; - -[[noreturn]] void Shutdown(void* parg) -{ - std::exit(EXIT_SUCCESS); -} - -[[noreturn]] void StartShutdown() -{ - std::exit(EXIT_SUCCESS); -} - -bool ShutdownRequested() -{ - return false; -} diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 474a67497f..b4c0e6a0f4 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <timedata.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp index c7ceb2f1e9..6d8459f5b1 100644 --- a/src/test/torcontrol_tests.cpp +++ b/src/test/torcontrol_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <torcontrol.h> #include <boost/test/unit_test.hpp> @@ -20,7 +20,6 @@ BOOST_FIXTURE_TEST_SUITE(torcontrol_tests, BasicTestingSetup) static void CheckSplitTorReplyLine(std::string input, std::string command, std::string args) { - BOOST_TEST_MESSAGE(std::string("CheckSplitTorReplyLine(") + input + ")"); auto ret = SplitTorReplyLine(input); BOOST_CHECK_EQUAL(ret.first, command); BOOST_CHECK_EQUAL(ret.second, args); @@ -61,7 +60,6 @@ BOOST_AUTO_TEST_CASE(util_SplitTorReplyLine) static void CheckParseTorReplyMapping(std::string input, std::map<std::string,std::string> expected) { - BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(") + input + ")"); auto ret = ParseTorReplyMapping(input); BOOST_CHECK_EQUAL(ret.size(), expected.size()); auto r_it = ret.begin(); @@ -173,7 +171,6 @@ BOOST_AUTO_TEST_CASE(util_ParseTorReplyMapping) // Special handling for null case // (needed because string comparison reads the null as end-of-string) - BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(Null=\"\\0\")")); auto ret = ParseTorReplyMapping("Null=\"\\0\""); BOOST_CHECK_EQUAL(ret.size(), 1U); auto r_it = ret.begin(); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 39cff3f463..6242fdabd1 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -1,20 +1,21 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/tx_invalid.json.h> #include <test/data/tx_valid.json.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <clientversion.h> #include <checkqueue.h> -#include <consensus/tx_verify.h> +#include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> #include <key.h> #include <keystore.h> #include <validation.h> #include <policy/policy.h> +#include <policy/settings.h> #include <script/script.h> #include <script/sign.h> #include <script/script_error.h> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index d667c26c3c..9d62b471c1 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> #include <index/txindex.h> #include <script/standard.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/system.h> #include <util/time.h> #include <validation.h> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index c2777cd6d1..26ae7be202 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> @@ -52,10 +52,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) // Check that the validation state reflects the unsuccessful attempt. BOOST_CHECK(state.IsInvalid()); BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); - - int nDoS; - BOOST_CHECK_EQUAL(state.IsInvalid(nDoS), true); - BOOST_CHECK_EQUAL(nDoS, 100); + BOOST_CHECK(state.GetReason() == ValidationInvalidReason::CONSENSUS); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 4d04aae7e9..01018043b1 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <random.h> #include <script/standard.h> #include <script/sign.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/time.h> #include <core_io.h> #include <keystore.h> diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index cca5e20296..c1749fb856 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <arith_uint256.h> #include <uint256.h> #include <version.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/test/unit_test.hpp> #include <stdint.h> diff --git a/src/test/util.cpp b/src/test/util.cpp new file mode 100644 index 0000000000..05d3a97a59 --- /dev/null +++ b/src/test/util.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util.h> + +#include <chainparams.h> +#include <consensus/merkle.h> +#include <consensus/validation.h> +#include <key_io.h> +#include <miner.h> +#include <outputtype.h> +#include <pow.h> +#include <scheduler.h> +#include <script/standard.h> +#include <txdb.h> +#include <validation.h> +#include <validationinterface.h> +#ifdef ENABLE_WALLET +#include <wallet/wallet.h> +#endif + +#include <boost/thread.hpp> + +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; + +#ifdef ENABLE_WALLET +std::string getnewaddress(CWallet& w) +{ + constexpr auto output_type = OutputType::BECH32; + + CPubKey new_key; + if (!w.GetKeyFromPool(new_key)) assert(false); + + w.LearnRelatedScripts(new_key, output_type); + const auto dest = GetDestinationForKey(new_key, output_type); + + w.SetAddressBook(dest, /* label */ "", "receive"); + + return EncodeDestination(dest); +} + +void importaddress(CWallet& wallet, const std::string& address) +{ + LOCK(wallet.cs_wallet); + const auto dest = DecodeDestination(address); + assert(IsValidDestination(dest)); + const auto script = GetScriptForDestination(dest); + wallet.MarkDirty(); + assert(!wallet.HaveWatchOnly(script)); + if (!wallet.AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); + wallet.SetAddressBook(dest, /* label */ "", "receive"); +} +#endif // ENABLE_WALLET + +CTxIn generatetoaddress(const std::string& address) +{ + const auto dest = DecodeDestination(address); + assert(IsValidDestination(dest)); + const auto coinbase_script = GetScriptForDestination(dest); + + return MineBlock(coinbase_script); +} + +CTxIn MineBlock(const CScript& coinbase_scriptPubKey) +{ + auto block = PrepareBlock(coinbase_scriptPubKey); + + while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { + ++block->nNonce; + assert(block->nNonce); + } + + bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; + assert(processed); + + return CTxIn{block->vtx[0]->GetHash(), 0}; +} + +std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey) +{ + auto block = std::make_shared<CBlock>( + BlockAssembler{Params()} + .CreateNewBlock(coinbase_scriptPubKey) + ->block); + + block->nTime = ::chainActive.Tip()->GetMedianTimePast() + 1; + block->hashMerkleRoot = BlockMerkleRoot(*block); + + return block; +} diff --git a/src/test/util.h b/src/test/util.h new file mode 100644 index 0000000000..8ba647ec3f --- /dev/null +++ b/src/test/util.h @@ -0,0 +1,38 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_H +#define BITCOIN_TEST_UTIL_H + +#include <memory> +#include <string> + +class CBlock; +class CScript; +class CTxIn; +class CWallet; + +// Constants // + +extern const std::string ADDRESS_BCRT1_UNSPENDABLE; + +// Lower-level utils // + +/** Returns the generated coin */ +CTxIn MineBlock(const CScript& coinbase_scriptPubKey); +/** Prepare a block to be mined */ +std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey); + + +// RPC-like // + +/** Import the address to the wallet */ +void importaddress(CWallet& wallet, const std::string& address); +/** Returns a new address from the wallet */ +std::string getnewaddress(CWallet& w); +/** Returns the generated coin */ +CTxIn generatetoaddress(const std::string& address); + + +#endif // BITCOIN_TEST_UTIL_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 860f64bb11..0f1834240d 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <sync.h> #include <util/strencodings.h> #include <util/moneystr.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <stdint.h> #include <vector> @@ -36,8 +36,10 @@ BOOST_AUTO_TEST_CASE(util_criticalsection) do { TRY_LOCK(cs, lockTest); - if (lockTest) + if (lockTest) { + BOOST_CHECK(true); // Needed to suppress "Test case [...] did not check any assertions" break; + } BOOST_ERROR("break was swallowed!"); } while(0); @@ -78,80 +80,40 @@ BOOST_AUTO_TEST_CASE(util_HexStr) "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected + 5, true), - "04 67 8a fd b0"); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_expected + sizeof(ParseHex_expected), ParseHex_expected + sizeof(ParseHex_expected)), ""); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected + sizeof(ParseHex_expected), - ParseHex_expected + sizeof(ParseHex_expected), true), - ""); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_expected, ParseHex_expected), ""); - BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected, true), - ""); - std::vector<unsigned char> ParseHex_vec(ParseHex_expected, ParseHex_expected + 5); BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec, true), - "04 67 8a fd b0"); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_vec.rbegin(), ParseHex_vec.rend()), "b0fd8a6704" ); BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec.rbegin(), ParseHex_vec.rend(), true), - "b0 fd 8a 67 04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 1), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "04" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 1), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 5), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "b0fd8a6704" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 5), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "b0 fd 8a 67 04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 65), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "5f1df16b2b704c8a578d0bbaf74d385cde12c11ee50455f3c438ef4c3fbcf649b6de611feae06279a60939e028a8d65c10b73071a6f16719274855feb0fd8a6704" @@ -180,9 +142,10 @@ struct TestArgsManager : public ArgsManager { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } std::string error; - BOOST_REQUIRE(ReadConfigStream(streamConfig, error)); + BOOST_REQUIRE(ReadConfigStream(streamConfig, "", error)); } void SetNetworkOnlyArg(const std::string arg) { @@ -195,6 +158,9 @@ struct TestArgsManager : public ArgsManager AddArg(args[i], "", false, OptionsCategory::OPTIONS); } } + using ArgsManager::ReadConfigStream; + using ArgsManager::cs_args; + using ArgsManager::m_network; }; BOOST_AUTO_TEST_CASE(util_ParseParameters) @@ -612,6 +578,232 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); } +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentially. +// +// The test covers: +// +// - Combining different setting actions. Possible actions are: configuring a +// setting, negating a setting (adding "-no" prefix), and configuring/negating +// settings in a network section (adding "main." or "test." prefixes). +// +// - Combining settings from command line arguments and a config file. +// +// - Combining SoftSet and ForceSet calls. +// +// - Testing "main" and "test" network values to make sure settings from network +// sections are applied and to check for mainnet-specific behaviors like +// inheriting settings from the default section. +// +// - Testing network-specific settings like "-wallet", that may be ignored +// outside a network section, and non-network specific settings like "-server" +// that aren't sensitive to the network. +// +struct SettingsMergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { SET = 0, NEGATE, SECTION_SET, SECTION_NEGATE, END }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ForEachActionList([&](const ActionList& arg_actions) { + ForEachActionList([&](const ActionList& conf_actions) { + for (bool soft_set : {false, true}) { + for (bool force_set : {false, true}) { + for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET}) { + for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET}) { + for (bool net_specific : {false, true}) { + fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); + } + } + } + } + } + }); + }); + } + + //! Enumerate interesting combinations of actions. + template <typename Fn> + void ForEachActionList(Fn&& fn) + { + ActionList actions = {SET}; + for (bool done = false; !done;) { + int prev_action = -1; + bool skip_actions = false; + for (Action action : actions) { + if ((prev_action == END && action != END) || (prev_action != END && action == prev_action)) { + // To cut down list of enumerated settings, skip enumerating + // settings with ignored actions after an END, and settings that + // repeat the same action twice in a row. + skip_actions = true; + break; + } + prev_action = action; + } + if (!skip_actions) fn(actions); + done = true; + for (Action& action : actions) { + action = Action(action < END ? action + 1 : 0); + if (action) { + done = false; + break; + } + } + } + } + + //! Translate actions into a list of <key>=<value> setting strings. + std::vector<std::string> GetValues(const ActionList& actions, + const std::string& section, + const std::string& name, + const std::string& value_prefix) + { + std::vector<std::string> values; + int suffix = 0; + for (Action action : actions) { + if (action == END) break; + std::string prefix; + if (action == SECTION_SET || action == SECTION_NEGATE) prefix = section + "."; + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + values.push_back(prefix + name + "=" + value_prefix + std::to_string(++suffix)); + } + } + if (action == NEGATE || action == SECTION_NEGATE) { + values.push_back(prefix + "no" + name + "=1"); + } + } + return values; + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see below). +BOOST_FIXTURE_TEST_CASE(util_SettingsMerge, SettingsMergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool soft_set, bool force_set, + const std::string& section, const std::string& network, bool net_specific) { + TestArgsManager parser; + LOCK(parser.cs_args); + + std::string desc = "net="; + desc += network; + parser.m_network = network; + + const std::string& name = net_specific ? "server" : "wallet"; + const std::string key = "-" + name; + parser.AddArg(key, name, false, OptionsCategory::OPTIONS); + if (net_specific) parser.SetNetworkOnlyArg(key); + + auto args = GetValues(arg_actions, section, name, "a"); + std::vector<const char*> argv = {"ignored"}; + for (auto& arg : args) { + arg.insert(0, "-"); + desc += " "; + desc += arg; + argv.push_back(arg.c_str()); + } + std::string error; + BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); + BOOST_CHECK_EQUAL(error, ""); + + std::string conf; + for (auto& conf_val : GetValues(conf_actions, section, name, "c")) { + desc += " "; + desc += conf_val; + conf += conf_val; + conf += "\n"; + } + std::istringstream conf_stream(conf); + BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); + BOOST_CHECK_EQUAL(error, ""); + + if (soft_set) { + desc += " soft"; + parser.SoftSetArg(key, "soft1"); + parser.SoftSetArg(key, "soft2"); + } + + if (force_set) { + desc += " force"; + parser.ForceSetArg(key, "force1"); + parser.ForceSetArg(key, "force2"); + } + + desc += " || "; + + if (!parser.IsArgSet(key)) { + desc += "unset"; + BOOST_CHECK(!parser.IsArgNegated(key)); + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "default"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else if (parser.IsArgNegated(key)) { + desc += "negated"; + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "0"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else { + desc += parser.GetArg(key, "default"); + desc += " |"; + for (const auto& arg : parser.GetArgs(key)) { + desc += " "; + desc += arg; + } + } + + std::set<std::string> ignored = parser.GetUnsuitableSectionOnlyArgs(); + if (!ignored.empty()) { + desc += " | ignored"; + for (const auto& arg : ignored) { + desc += " "; + desc += arg; + } + } + + desc += "\n"; + + out_sha.Write((const unsigned char*)desc.data(), desc.size()); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + + // If check below fails, should manually dump the results with: + // + // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_SettingsMerge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || <IsArgSet/IsArgNegated/GetArg output> | <GetArgs output> | <GetUnsuitable output> + BOOST_CHECK_EQUAL(out_sha_hex, "80964e17fbd3c5569d3c824d032e28e2d319ef57494735b0e76eb7aad9957f2c"); +} + BOOST_AUTO_TEST_CASE(util_FormatMoney) { BOOST_CHECK_EQUAL(FormatMoney(0), "0.00"); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp new file mode 100644 index 0000000000..71c0168ca3 --- /dev/null +++ b/src/test/util_threadnames_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/threadnames.h> +#include <test/setup_common.h> + +#include <thread> +#include <vector> +#include <set> +#include <mutex> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(util_threadnames_tests, BasicTestingSetup) + +const std::string TEST_THREAD_NAME_BASE = "test_thread."; + +/** + * Run a bunch of threads to all call util::ThreadRename. + * + * @return the set of name each thread has after attempted renaming. + */ +std::set<std::string> RenameEnMasse(int num_threads) +{ + std::vector<std::thread> threads; + std::set<std::string> names; + std::mutex lock; + + auto RenameThisThread = [&](int i) { + util::ThreadRename(TEST_THREAD_NAME_BASE + std::to_string(i)); + std::lock_guard<std::mutex> guard(lock); + names.insert(util::ThreadGetInternalName()); + }; + + for (int i = 0; i < num_threads; ++i) { + threads.push_back(std::thread(RenameThisThread, i)); + } + + for (std::thread& thread : threads) thread.join(); + + return names; +} + +/** + * Rename a bunch of threads with the same basename (expect_multiple=true), ensuring suffixes are + * applied properly. + */ +BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) +{ + BOOST_CHECK_EQUAL(util::ThreadGetInternalName(), ""); + +#if !defined(HAVE_THREAD_LOCAL) + // This test doesn't apply to platforms where we don't have thread_local. + return; +#endif + + std::set<std::string> names = RenameEnMasse(100); + + BOOST_CHECK_EQUAL(names.size(), 100); + + // Names "test_thread.[n]" should exist for n = [0, 99] + for (int i = 0; i < 100; ++i) { + BOOST_CHECK(names.find(TEST_THREAD_NAME_BASE + std::to_string(i)) != names.end()); + } + +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 44432cd0a1..4d54aa9b2c 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ #include <miner.h> #include <pow.h> #include <random.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/test/main_tests.cpp b/src/test/validation_tests.cpp index 5b3f2bc578..101025d31e 100644 --- a/src/test/main_tests.cpp +++ b/src/test/validation_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <validation.h> #include <net.h> +#include <validation.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(main_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_tests, TestingSetup) static void TestBlockSubsidyHalvings(const Consensus::Params& consensusParams) { diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index ca3196454a..38d91b6647 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> #include <versionbits.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <chainparams.h> #include <validation.h> #include <consensus/params.h> diff --git a/src/txdb.h b/src/txdb.h index d5a4bfaeb1..c4ece11503 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -37,6 +37,8 @@ static const int64_t nMaxBlockDBCache = 2; // Unlike for the UTXO database, for the txindex scenario the leveldb cache make // a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991 static const int64_t nMaxTxIndexCache = 1024; +//! Max memory allocated to all block filter index caches combined in MiB. +static const int64_t max_filter_index_cache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 68f47d5cce..90b28227a0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -11,6 +11,7 @@ #include <validation.h> #include <policy/policy.h> #include <policy/fees.h> +#include <policy/settings.h> #include <reverse_iterator.h> #include <streams.h> #include <timedata.h> @@ -764,7 +765,7 @@ std::vector<CTxMemPool::indexed_transaction_set::const_iterator> CTxMemPool::Get return iters; } -void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) +void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) const { LOCK(cs); auto iters = GetSortedDepthAndScore(); @@ -1090,4 +1091,16 @@ void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, } } +bool CTxMemPool::IsLoaded() const +{ + LOCK(cs); + return m_is_loaded; +} + +void CTxMemPool::SetIsLoaded(bool loaded) +{ + LOCK(cs); + m_is_loaded = loaded; +} + SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index f7afaec8fc..3ada47a28e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -184,7 +184,7 @@ private: const LockPoints& lp; }; -// extracts a transaction hash from CTxMempoolEntry or CTransactionRef +// extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -455,6 +455,8 @@ private: void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool m_is_loaded GUARDED_BY(cs){false}; + public: static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; // public only for testing @@ -588,7 +590,7 @@ public: void clear(); void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); - void queryHashes(std::vector<uint256>& vtxid); + void queryHashes(std::vector<uint256>& vtxid) const; bool isSpent(const COutPoint& outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); @@ -672,6 +674,12 @@ public: */ void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const; + /** @returns true if the mempool is fully loaded */ + bool IsLoaded() const; + + /** Sets the current loaded state */ + void SetIsLoaded(bool loaded); + unsigned long size() const { LOCK(cs); diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 947d7e2308..31a95486d7 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -28,10 +28,6 @@ struct UISignals { boost::signals2::connection CClientUIInterface::signal_name##_connect(std::function<signal_name##Sig> fn) \ { \ return g_ui_signals.signal_name.connect(fn); \ - } \ - void CClientUIInterface::signal_name##_disconnect(std::function<signal_name##Sig> fn) \ - { \ - return g_ui_signals.signal_name.disconnect(&fn); \ } ADD_SIGNALS_IMPL_WRAPPER(ThreadSafeMessageBox); @@ -52,7 +48,7 @@ void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_s void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } -void CClientUIInterface::LoadWallet(std::shared_ptr<CWallet> wallet) { return g_ui_signals.LoadWallet(wallet); } +void CClientUIInterface::LoadWallet(std::unique_ptr<interfaces::Wallet>& wallet) { return g_ui_signals.LoadWallet(wallet); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } @@ -69,13 +65,3 @@ void InitWarning(const std::string& str) { uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); } - -std::string AmountHighWarn(const std::string& optname) -{ - return strprintf(_("%s is set very high!"), optname); -} - -std::string AmountErrMsg(const char* const optname, const std::string& strValue) -{ - return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); -} diff --git a/src/ui_interface.h b/src/ui_interface.h index fe466b3ca4..d408f6f889 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -11,7 +11,6 @@ #include <stdint.h> #include <string> -class CWallet; class CBlockIndex; namespace boost { namespace signals2 { @@ -19,6 +18,10 @@ class connection; } } // namespace boost +namespace interfaces { +class Wallet; +} // namespace interfaces + /** General change type (added, updated, removed). */ enum ChangeType { @@ -78,8 +81,7 @@ public: #define ADD_SIGNALS_DECL_WRAPPER(signal_name, rtype, ...) \ rtype signal_name(__VA_ARGS__); \ using signal_name##Sig = rtype(__VA_ARGS__); \ - boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); \ - void signal_name##_disconnect(std::function<signal_name##Sig> fn); + boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); /** Show message box. */ ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const std::string& message, const std::string& caption, unsigned int style); @@ -102,7 +104,7 @@ public: ADD_SIGNALS_DECL_WRAPPER(NotifyAlertChanged, void, ); /** A wallet has been loaded. */ - ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::shared_ptr<CWallet> wallet); + ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::unique_ptr<interfaces::Wallet>& wallet); /** * Show progress e.g. for verifychain. @@ -126,10 +128,6 @@ void InitWarning(const std::string& str); /** Show error message **/ bool InitError(const std::string& str); -std::string AmountHighWarn(const std::string& optname); - -std::string AmountErrMsg(const char* const optname, const std::string& strValue); - extern CClientUIInterface uiInterface; #endif // BITCOIN_UI_INTERFACE_H diff --git a/src/util/error.cpp b/src/util/error.cpp new file mode 100644 index 0000000000..68ffd8b046 --- /dev/null +++ b/src/util/error.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2010-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/error.h> + +#include <util/system.h> + +std::string TransactionErrorString(const TransactionError err) +{ + switch (err) { + case TransactionError::OK: + return "No error"; + case TransactionError::MISSING_INPUTS: + return "Missing inputs"; + case TransactionError::ALREADY_IN_CHAIN: + return "Transaction already in block chain"; + case TransactionError::P2P_DISABLED: + return "Peer-to-peer functionality missing or disabled"; + case TransactionError::MEMPOOL_REJECTED: + return "Transaction rejected by AcceptToMemoryPool"; + case TransactionError::MEMPOOL_ERROR: + return "AcceptToMemoryPool failed"; + case TransactionError::INVALID_PSBT: + return "PSBT is not sane"; + case TransactionError::PSBT_MISMATCH: + return "PSBTs not compatible (different transactions)"; + case TransactionError::SIGHASH_MISMATCH: + return "Specified sighash value does not match existing value"; + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +std::string AmountHighWarn(const std::string& optname) +{ + return strprintf(_("%s is set very high!"), optname); +} + +std::string AmountErrMsg(const char* const optname, const std::string& strValue) +{ + return strprintf(_("Invalid amount for -%s=<amount>: '%s'"), optname, strValue); +} diff --git a/src/util/error.h b/src/util/error.h new file mode 100644 index 0000000000..d93309551b --- /dev/null +++ b/src/util/error.h @@ -0,0 +1,38 @@ +// Copyright (c) 2010-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ERROR_H +#define BITCOIN_UTIL_ERROR_H + +/** + * util/error.h is a common place for definitions of simple error types and + * string functions. Types and functions defined here should not require any + * outside dependencies. + * + * Error types defined here can be used in different parts of the bitcoin + * codebase, to avoid the need to write boilerplate code catching and + * translating errors passed across wallet/node/rpc/gui code boundaries. + */ + +#include <string> + +enum class TransactionError { + OK, //!< No error + MISSING_INPUTS, + ALREADY_IN_CHAIN, + P2P_DISABLED, + MEMPOOL_REJECTED, + MEMPOOL_ERROR, + INVALID_PSBT, + PSBT_MISMATCH, + SIGHASH_MISMATCH, +}; + +std::string TransactionErrorString(const TransactionError error); + +std::string AmountHighWarn(const std::string& optname); + +std::string AmountErrMsg(const char* const optname, const std::string& strValue); + +#endif // BITCOIN_UTIL_ERROR_H diff --git a/src/util/fees.cpp b/src/util/fees.cpp new file mode 100644 index 0000000000..5fdaa1284c --- /dev/null +++ b/src/util/fees.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/fees.h> + +#include <string> + +std::string StringForFeeReason(FeeReason reason) { + static const std::map<FeeReason, std::string> fee_reason_strings = { + {FeeReason::NONE, "None"}, + {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, + {FeeReason::FULL_ESTIMATE, "Target 85% Threshold"}, + {FeeReason::DOUBLE_ESTIMATE, "Double Target 95% Threshold"}, + {FeeReason::CONSERVATIVE, "Conservative Double Target longer horizon"}, + {FeeReason::MEMPOOL_MIN, "Mempool Min Fee"}, + {FeeReason::PAYTXFEE, "PayTxFee set"}, + {FeeReason::FALLBACK, "Fallback fee"}, + {FeeReason::REQUIRED, "Minimum Required Fee"}, + {FeeReason::MAXTXFEE, "MaxTxFee limit"} + }; + auto reason_string = fee_reason_strings.find(reason); + + if (reason_string == fee_reason_strings.end()) return "Unknown"; + + return reason_string->second; +} + +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) { + static const std::map<std::string, FeeEstimateMode> fee_modes = { + {"UNSET", FeeEstimateMode::UNSET}, + {"ECONOMICAL", FeeEstimateMode::ECONOMICAL}, + {"CONSERVATIVE", FeeEstimateMode::CONSERVATIVE}, + }; + auto mode = fee_modes.find(mode_string); + + if (mode == fee_modes.end()) return false; + + fee_estimate_mode = mode->second; + return true; +} diff --git a/src/util/fees.h b/src/util/fees.h new file mode 100644 index 0000000000..fc355ce9c2 --- /dev/null +++ b/src/util/fees.h @@ -0,0 +1,16 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_UTIL_FEES_H +#define BITCOIN_UTIL_FEES_H + +#include <string> + +enum class FeeEstimateMode; +enum class FeeReason; + +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); +std::string StringForFeeReason(FeeReason reason); + +#endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/rbf.cpp b/src/util/rbf.cpp new file mode 100644 index 0000000000..d520a9606d --- /dev/null +++ b/src/util/rbf.cpp @@ -0,0 +1,17 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/rbf.h> + +#include <primitives/transaction.h> + +bool SignalsOptInRBF(const CTransaction &tx) +{ + for (const CTxIn &txin : tx.vin) { + if (txin.nSequence <= MAX_BIP125_RBF_SEQUENCE) { + return true; + } + } + return false; +} diff --git a/src/util/rbf.h b/src/util/rbf.h new file mode 100644 index 0000000000..d3ef110628 --- /dev/null +++ b/src/util/rbf.h @@ -0,0 +1,18 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_RBF_H +#define BITCOIN_UTIL_RBF_H + +#include <cstdint> + +class CTransaction; + +static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; + +// Check whether the sequence numbers on this transaction are signaling +// opt-in to replace-by-fee, according to BIP 125 +bool SignalsOptInRBF(const CTransaction &tx); + +#endif // BITCOIN_UTIL_RBF_H diff --git a/src/util/strencodings.h b/src/util/strencodings.h index cf77044094..7c4364a082 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -12,6 +12,7 @@ #include <attributes.h> #include <cstdint> +#include <iterator> #include <string> #include <vector> @@ -121,28 +122,25 @@ NODISCARD bool ParseUInt64(const std::string& str, uint64_t *out); NODISCARD bool ParseDouble(const std::string& str, double *out); template<typename T> -std::string HexStr(const T itbegin, const T itend, bool fSpaces=false) +std::string HexStr(const T itbegin, const T itend) { std::string rv; static const char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - rv.reserve((itend-itbegin)*3); + rv.reserve(std::distance(itbegin, itend) * 2); for(T it = itbegin; it < itend; ++it) { unsigned char val = (unsigned char)(*it); - if(fSpaces && it != itbegin) - rv.push_back(' '); rv.push_back(hexmap[val>>4]); rv.push_back(hexmap[val&15]); } - return rv; } template<typename T> -inline std::string HexStr(const T& vch, bool fSpaces=false) +inline std::string HexStr(const T& vch) { - return HexStr(vch.begin(), vch.end(), fSpaces); + return HexStr(vch.begin(), vch.end()); } /** diff --git a/src/util/system.cpp b/src/util/system.cpp index 6e82de743a..6925bda4ef 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -60,10 +60,6 @@ #include <shlobj.h> #endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif - #ifdef HAVE_MALLOPT_ARENA_MAX #include <malloc.h> #endif @@ -135,6 +131,14 @@ bool DirIsWritable(const fs::path& directory) return true; } +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) +{ + constexpr uint64_t min_disk_space = 52428800; // 50 MiB + + uint64_t free_bytes_available = fs::space(dir).available; + return free_bytes_available >= min_disk_space + additional_bytes; +} + /** * Interpret a string argument as a boolean. * @@ -353,8 +357,7 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const return unsuitables; } - -const std::set<std::string> ArgsManager::GetUnrecognizedSections() const +const std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ @@ -362,14 +365,11 @@ const std::set<std::string> ArgsManager::GetUnrecognizedSections() const CBaseChainParams::TESTNET, CBaseChainParams::MAIN }; - std::set<std::string> diff; LOCK(cs_args); - std::set_difference( - m_config_sections.begin(), m_config_sections.end(), - available_sections.begin(), available_sections.end(), - std::inserter(diff, diff.end())); - return diff; + std::list<SectionInfo> unrecognized = m_config_sections; + unrecognized.remove_if([](const SectionInfo& appeared){ return available_sections.find(appeared.m_name) != available_sections.end(); }); + return unrecognized; } void ArgsManager::SelectConfigNetwork(const std::string& network) @@ -793,7 +793,7 @@ static std::string TrimString(const std::string& str, const std::string& pattern return str.substr(front, end - front + 1); } -static bool GetConfigOptions(std::istream& stream, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::set<std::string>& sections) +static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; std::string::size_type pos; @@ -809,7 +809,7 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect if (!str.empty()) { if (*str.begin() == '[' && *str.rbegin() == ']') { const std::string section = str.substr(1, str.size() - 2); - sections.insert(section); + sections.emplace_back(SectionInfo{section, filepath, linenr}); prefix = section + '.'; } else if (*str.begin() == '-') { error = strprintf("parse error on line %i: %s, options in configuration file must be specified without leading -", linenr, str); @@ -822,8 +822,8 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return false; } options.emplace_back(name, value); - if ((pos = name.rfind('.')) != std::string::npos) { - sections.insert(name.substr(0, pos)); + if ((pos = name.rfind('.')) != std::string::npos && prefix.length() <= pos) { + sections.emplace_back(SectionInfo{name.substr(0, pos), filepath, linenr}); } } else { error = strprintf("parse error on line %i: %s", linenr, str); @@ -838,12 +838,11 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return true; } -bool ArgsManager::ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys) +bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); std::vector<std::pair<std::string, std::string>> options; - m_config_sections.clear(); - if (!GetConfigOptions(stream, error, options, m_config_sections)) { + if (!GetConfigOptions(stream, filepath, error, options, m_config_sections)) { return false; } for (const std::pair<std::string, std::string>& option : options) { @@ -874,6 +873,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); @@ -881,7 +881,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) // ok to not have a config file if (stream.good()) { - if (!ReadConfigStream(stream, error, ignore_invalid_keys)) { + if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } // if there is an -includeconf in the override args, but it is empty, that means the user @@ -912,7 +912,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) for (const std::string& to_include : includeconf) { fsbridge::ifstream include_config(GetConfigFile(to_include)); if (include_config.good()) { - if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) { + if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { return false; } LogPrintf("Included configuration file %s\n", to_include.c_str()); @@ -1085,11 +1085,12 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { fcntl(fileno(file), F_PREALLOCATE, &fst); } ftruncate(fileno(file), fst.fst_length); -#elif defined(__linux__) +#else + #if defined(__linux__) // Version using posix_fallocate off_t nEndPos = (off_t)offset + length; - posix_fallocate(fileno(file), 0, nEndPos); -#else + if (0 == posix_fallocate(fileno(file), 0, nEndPos)) return; + #endif // Fallback version // TODO: just write one byte per block static const char buf[65536] = {}; @@ -1133,22 +1134,6 @@ void runCommand(const std::string& strCommand) LogPrintf("runCommand error: system(%s) returned %d\n", strCommand, nErr); } -void RenameThread(const char* name) -{ -#if defined(PR_SET_NAME) - // Only the first 15 characters are used (16 - NUL terminator) - ::prctl(PR_SET_NAME, name, 0, 0, 0); -#elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) - pthread_set_name_np(pthread_self(), name); - -#elif defined(MAC_OSX) - pthread_setname_np(name); -#else - // Prevent warnings for unused parameters... - (void)name; -#endif -} - void SetupEnvironment() { #ifdef HAVE_MALLOPT_ARENA_MAX diff --git a/src/util/system.h b/src/util/system.h index 69ae11d1ed..1a83cb67b1 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -20,6 +20,7 @@ #include <fs.h> #include <logging.h> #include <sync.h> +#include <util/threadnames.h> #include <tinyformat.h> #include <util/memory.h> #include <util/time.h> @@ -72,6 +73,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. @@ -127,6 +129,13 @@ enum class OptionsCategory { HIDDEN // Always the last option to avoid printing these in the help }; +struct SectionInfo +{ + std::string m_name; + std::string m_file; + int m_line; +}; + class ArgsManager { protected: @@ -147,9 +156,9 @@ protected: std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); - std::set<std::string> m_config_sections GUARDED_BY(cs_args); + std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); - NODISCARD bool ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys = false); + NODISCARD bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); public: ArgsManager(); @@ -173,7 +182,7 @@ public: /** * Log warnings for unrecognized section names in the config file. */ - const std::set<std::string> GetUnrecognizedSections() const; + const std::list<SectionInfo> GetUnrecognizedSections() const; /** * Return a vector of strings of the given argument @@ -317,15 +326,12 @@ std::string HelpMessageOpt(const std::string& option, const std::string& message */ int GetNumCores(); -void RenameThread(const char* name); - /** * .. and a wrapper that just calls func once */ template <typename Callable> void TraceThread(const char* name, Callable func) { - std::string s = strprintf("bitcoin-%s", name); - RenameThread(s.c_str()); + util::ThreadRename(name); try { LogPrintf("%s thread start\n", name); diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp new file mode 100644 index 0000000000..7b0d744aec --- /dev/null +++ b/src/util/threadnames.cpp @@ -0,0 +1,57 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <atomic> +#include <thread> + +#include <util/threadnames.h> + +#ifdef HAVE_SYS_PRCTL_H +#include <sys/prctl.h> // For prctl, PR_SET_NAME, PR_GET_NAME +#endif + +//! Set the thread's name at the process level. Does not affect the +//! internal name. +static void SetThreadName(const char* name) +{ +#if defined(PR_SET_NAME) + // Only the first 15 characters are used (16 - NUL terminator) + ::prctl(PR_SET_NAME, name, 0, 0, 0); +#elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) + pthread_set_name_np(pthread_self(), name); +#elif defined(MAC_OSX) + pthread_setname_np(name); +#else + // Prevent warnings for unused parameters... + (void)name; +#endif +} + +// If we have thread_local, just keep thread ID and name in a thread_local +// global. +#if defined(HAVE_THREAD_LOCAL) + +static thread_local std::string g_thread_name; +const std::string& util::ThreadGetInternalName() { return g_thread_name; } +//! Set the in-memory internal name for this thread. Does not affect the process +//! name. +static void SetInternalName(std::string name) { g_thread_name = std::move(name); } + +// Without thread_local available, don't handle internal name at all. +#else + +static const std::string empty_string; +const std::string& util::ThreadGetInternalName() { return empty_string; } +static void SetInternalName(std::string name) { } +#endif + +void util::ThreadRename(std::string&& name) +{ + SetThreadName(("bitcoin-" + name).c_str()); + SetInternalName(std::move(name)); +} diff --git a/src/util/threadnames.h b/src/util/threadnames.h new file mode 100644 index 0000000000..aaf07b9bf8 --- /dev/null +++ b/src/util/threadnames.h @@ -0,0 +1,21 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_THREADNAMES_H +#define BITCOIN_UTIL_THREADNAMES_H + +#include <string> + +namespace util { +//! Rename a thread both in terms of an internal (in-memory) name as well +//! as its system thread name. +void ThreadRename(std::string&&); + +//! Get the thread's internal (in-memory) name; used e.g. for identification in +//! logging. +const std::string& ThreadGetInternalName(); + +} // namespace util + +#endif // BITCOIN_UTIL_THREADNAMES_H diff --git a/src/util/url.cpp b/src/util/url.cpp new file mode 100644 index 0000000000..49eacbf2d0 --- /dev/null +++ b/src/util/url.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2015-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/url.h> + +#include <event2/http.h> +#include <stdlib.h> +#include <string> + +std::string urlDecode(const std::string &urlEncoded) { + std::string res; + if (!urlEncoded.empty()) { + char *decoded = evhttp_uridecode(urlEncoded.c_str(), false, nullptr); + if (decoded) { + res = std::string(decoded); + free(decoded); + } + } + return res; +} diff --git a/src/util/url.h b/src/util/url.h new file mode 100644 index 0000000000..3d7315a338 --- /dev/null +++ b/src/util/url.h @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_URL_H +#define BITCOIN_UTIL_URL_H + +#include <string> + +std::string urlDecode(const std::string &urlEncoded); + +#endif // BITCOIN_UTIL_URL_H diff --git a/src/util/validation.cpp b/src/util/validation.cpp new file mode 100644 index 0000000000..fe1f5a277e --- /dev/null +++ b/src/util/validation.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/validation.h> + +#include <consensus/validation.h> +#include <tinyformat.h> + +/** Convert CValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const CValidationState &state) +{ + return strprintf("%s%s (code %i)", + state.GetRejectReason(), + state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage(), + state.GetRejectCode()); +} + +const std::string strMessageMagic = "Bitcoin Signed Message:\n"; diff --git a/src/util/validation.h b/src/util/validation.h new file mode 100644 index 0000000000..32559853ee --- /dev/null +++ b/src/util/validation.h @@ -0,0 +1,18 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_VALIDATION_H +#define BITCOIN_UTIL_VALIDATION_H + +#include <string> + +class CValidationState; + +/** Convert CValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const CValidationState &state); + +extern const std::string strMessageMagic; + +#endif // BITCOIN_UTIL_VALIDATION_H diff --git a/src/validation.cpp b/src/validation.cpp index f3d34dca70..ba9d6184ee 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -8,18 +8,20 @@ #include <arith_uint256.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <checkqueue.h> #include <consensus/consensus.h> #include <consensus/merkle.h> +#include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <cuckoocache.h> +#include <flatfile.h> #include <hash.h> #include <index/txindex.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> +#include <policy/settings.h> #include <pow.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -34,15 +36,19 @@ #include <txdb.h> #include <txmempool.h> #include <ui_interface.h> +#include <uint256.h> #include <undo.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/rbf.h> #include <util/strencodings.h> +#include <util/system.h> +#include <util/validation.h> #include <validationinterface.h> #include <warnings.h> #include <future> #include <sstream> +#include <string> #include <boost/algorithm/string/replace.hpp> #include <boost/thread.hpp> @@ -165,7 +171,7 @@ public: * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. */ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); @@ -177,7 +183,7 @@ public: // Manual block validity manipulation: bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); - bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool ReplayBlocks(const CChainParams& params, CCoinsView* view); @@ -204,10 +210,12 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Mark a block as not having block data + void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); } g_chainstate; /** @@ -233,7 +241,6 @@ std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); bool fHavePruned = false; bool fPruneMode = false; -bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; @@ -246,17 +253,13 @@ uint256 hashAssumeValid; arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); -std::atomic_bool g_is_mempool_loaded{false}; /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; -const std::string strMessageMagic = "Bitcoin Signed Message:\n"; - // Internal stuff namespace { CBlockIndex *&pindexBestInvalid = g_chainstate.pindexBestInvalid; @@ -317,7 +320,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState & static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -456,15 +461,6 @@ static void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) pcoinsTip->Uncache(removed); } -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state) -{ - return strprintf("%s%s (code %i)", - state.GetRejectReason(), - state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage(), - state.GetRejectCode()); -} - static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); @@ -565,6 +561,13 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationSt return CheckInputs(tx, state, view, true, flags, cacheSigStore, true, txdata); } +/** + * @param[out] coins_to_uncache Return any outpoints which were not previously present in the + * coins cache, but were added as a result of validating the tx + * for mempool acceptance. This allows the caller to optionally + * remove the cache additions if the associated transaction ends + * up being rejected by the mempool. + */ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -582,28 +585,28 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "coinbase"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "coinbase"); // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; if (fRequireStandard && !IsStandardTx(tx, reason)) - return state.DoS(0, false, REJECT_NONSTANDARD, reason); + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, reason); // Do not work on transactions that are too small. // A transaction with 1 segwit input and 1 P2WPHK output has non-witness size of 82 bytes. // Transactions smaller than this are not relayed to reduce unnecessary malloc overhead. if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) - return state.DoS(0, false, REJECT_NONSTANDARD, "tx-size-small"); + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) - return state.DoS(0, false, REJECT_NONSTANDARD, "non-final"); + return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_NONSTANDARD, "non-final"); // is it already in the memory pool? if (pool.exists(hash)) { - return state.Invalid(false, REJECT_DUPLICATE, "txn-already-in-mempool"); + return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, REJECT_DUPLICATE, "txn-already-in-mempool"); } // Check for conflicts with in-memory transactions @@ -639,7 +642,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool } } if (fReplacementOptOut) { - return state.Invalid(false, REJECT_DUPLICATE, "txn-mempool-conflict"); + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_DUPLICATE, "txn-mempool-conflict"); } setConflicts.insert(ptxConflicting->GetHash()); @@ -660,12 +663,16 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { coins_to_uncache.push_back(txin.prevout); } + + // Note: this call may add txin.prevout to the coins cache + // (pcoinsTip.cacheCoins) by way of FetchCoin(). It should be removed + // later (via coins_to_uncache) if this tx turns out to be invalid. if (!view.HaveCoin(txin.prevout)) { // Are inputs missing because we already have the tx? for (size_t out = 0; out < tx.vout.size(); out++) { // Optimistically just do efficient check of cache for outputs if (pcoinsTip->HaveCoinInCache(COutPoint(hash, out))) { - return state.Invalid(false, REJECT_DUPLICATE, "txn-already-known"); + return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, REJECT_DUPLICATE, "txn-already-known"); } } // Otherwise assume this might be an orphan tx for which we just haven't seen parents yet @@ -688,7 +695,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // Must keep pool.cs for this unless we change CheckSequenceLocks to take a // CoinsViewCache instead of create its own if (!CheckSequenceLocks(pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) - return state.DoS(0, false, REJECT_NONSTANDARD, "non-BIP68-final"); + return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_NONSTANDARD, "non-BIP68-final"); CAmount nFees = 0; if (!Consensus::CheckTxInputs(tx, state, view, GetSpendHeight(view), nFees)) { @@ -697,11 +704,11 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // Check for non-standard pay-to-script-hash in inputs if (fRequireStandard && !AreInputsStandard(tx, view)) - return state.Invalid(false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); // Check for non-standard witness in P2WSH if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, view)) - return state.DoS(0, false, REJECT_NONSTANDARD, "bad-witness-nonstandard", true); + return state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, REJECT_NONSTANDARD, "bad-witness-nonstandard"); int64_t nSigOpsCost = GetTransactionSigOpCost(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); @@ -724,27 +731,22 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); - // Check that the transaction doesn't have an excessive number of - // sigops, making it impossible to mine. Since the coinbase transaction - // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than - // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than - // merely non-standard transaction. if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) - return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", strprintf("%d", nSigOpsCost)); CAmount mempoolRejectFee = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); if (!bypass_limits && mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nModifiedFees, mempoolRejectFee)); + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", strprintf("%d < %d", nModifiedFees, mempoolRejectFee)); } // No transactions are allowed below minRelayTxFee except from disconnected blocks if (!bypass_limits && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) { - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "min relay fee not met", false, strprintf("%d < %d", nModifiedFees, ::minRelayTxFee.GetFee(nSize))); + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "min relay fee not met", strprintf("%d < %d", nModifiedFees, ::minRelayTxFee.GetFee(nSize))); } if (nAbsurdFee && nFees > nAbsurdFee) - return state.Invalid(false, + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_HIGHFEE, "absurdly-high-fee", strprintf("%d > %d", nFees, nAbsurdFee)); @@ -756,7 +758,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; std::string errString; if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { - return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString); + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "too-long-mempool-chain", errString); } // A transaction that spends outputs that would be replaced by it is invalid. Now @@ -768,8 +770,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); if (setConflicts.count(hashAncestor)) { - return state.DoS(10, false, - REJECT_INVALID, "bad-txns-spends-conflicting-tx", false, + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-spends-conflicting-tx", strprintf("%s spends conflicting transaction %s", hash.ToString(), hashAncestor.ToString())); @@ -811,8 +812,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); if (newFeeRate <= oldFeeRate) { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", hash.ToString(), newFeeRate.ToString(), @@ -840,8 +840,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool nConflictingSize += it->GetTxSize(); } } else { - return state.DoS(0, false, - REJECT_NONSTANDARD, "too many potential replacements", false, + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "too many potential replacements", strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", hash.ToString(), nConflictingCount, @@ -860,8 +859,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // it's cheaper to just check if the new input refers to a // tx that's in the mempool. if (pool.exists(tx.vin[j].prevout.hash)) { - return state.DoS(0, false, - REJECT_NONSTANDARD, "replacement-adds-unconfirmed", false, + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "replacement-adds-unconfirmed", strprintf("replacement %s adds unconfirmed input, idx %d", hash.ToString(), j)); } @@ -873,8 +871,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // transactions would not be paid for. if (nModifiedFees < nConflictingFees) { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); } @@ -884,8 +881,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool CAmount nDeltaFees = nModifiedFees - nConflictingFees; if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", hash.ToString(), FormatMoney(nDeltaFees), @@ -906,8 +902,10 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool if (!tx.HasWitness() && CheckInputs(tx, stateDummy, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && !CheckInputs(tx, stateDummy, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { // Only the witness is missing, so the transaction itself may be fine. - state.SetCorruptionPossible(); + state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, + state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); } + assert(IsTransactionReason(state.GetReason())); return false; // state filled in by CheckInputs } @@ -964,7 +962,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool if (!bypass_limits) { LimitMempoolSize(pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); if (!pool.exists(hash)) - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); + return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "mempool full"); } } @@ -981,6 +979,11 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo std::vector<COutPoint> coins_to_uncache; bool res = AcceptToMemoryPoolWorker(chainparams, pool, state, tx, pfMissingInputs, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept); if (!res) { + // Remove coins that were not present in the coins cache before calling ATMPW; + // this is to prevent memory DoS in case we receive a large number of + // invalid transactions that attempt to overrun the in-memory coins cache + // (`CCoinsViewCache::cacheCoins`). + for (const COutPoint& hashTx : coins_to_uncache) pcoinsTip->Uncache(hashTx); } @@ -1042,7 +1045,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus // CBlock and CBlockIndex // -static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1063,7 +1066,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1089,7 +1092,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - CDiskBlockPos blockPos; + FlatFilePos blockPos; { LOCK(cs_main); blockPos = pindex->GetBlockPos(); @@ -1103,9 +1106,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos hpos = pos; + FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { @@ -1140,7 +1143,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos block_pos; + FlatFilePos block_pos; { LOCK(cs_main); block_pos = pindex->GetBlockPos(); @@ -1297,7 +1300,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c } void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { - if (!state.CorruptionPossible()) { + if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); @@ -1365,6 +1368,9 @@ void InitScriptExecutionCache() { * which are matched. This is useful for checking blocks where we will likely never need the cache * entry again. * + * Note that we may set state.reason to NOT_STANDARD for extra soft-fork flags in flags, block-checking + * callers should probably reset it to CONSENSUS in such cases. + * * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp */ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -1420,22 +1426,26 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // Check whether the failure was caused by a // non-mandatory script verification check, such as // non-standard DER encodings or non-null dummy - // arguments; if so, don't trigger DoS protection to - // avoid splitting the network between upgraded and - // non-upgraded nodes. + // arguments; if so, ensure we return NOT_STANDARD + // instead of CONSENSUS to avoid downstream users + // splitting the network between upgraded and + // non-upgraded nodes by banning CONSENSUS-failing + // data providers. CScriptCheck check2(coin.out, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); if (check2()) - return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } - // Failures of other flags indicate a transaction that is - // invalid in new blocks, e.g. an invalid P2SH. We DoS ban - // such nodes as they are not following the protocol. That - // said during an upgrade careful thought should be taken - // as to the correct behavior - we may want to continue - // peering with non-upgraded nodes even after soft-fork - // super-majority signaling has occurred. - return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); + // MANDATORY flag failures correspond to + // ValidationInvalidReason::CONSENSUS. Because CONSENSUS + // failures are the most serious case of validation + // failures, we may need to consider using + // RECENT_CONSENSUS_CHANGE for any script failure that + // could be due to non-upgraded nodes which we may want to + // support, to avoid splitting the network (but this + // depends on the details of how net_processing handles + // such errors). + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -1450,9 +1460,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi return true; } -namespace { - -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1479,9 +1487,9 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return true; } -static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) { - CDiskBlockPos pos = pindex->GetUndoPos(); + FlatFilePos pos = pindex->GetUndoPos(); if (pos.IsNull()) { return error("%s: no undo data available", __func__); } @@ -1528,8 +1536,6 @@ static bool AbortNode(CValidationState& state, const std::string& strMessage, co return state.Error(strMessage); } -} // namespace - /** * Restore the UTXO in a Coin at a given COutPoint * @param undo The Coin to be restored. @@ -1627,37 +1633,24 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; + FlatFilePos _pos; if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) @@ -1674,8 +1667,8 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck() { - RenameThread("bitcoin-scriptch"); +void ThreadScriptCheck(int worker_num) { + util::ThreadRename(strprintf("scriptch.%i", worker_num)); scriptcheckqueue.Thread(); } @@ -1812,7 +1805,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // re-enforce that rule here (at least until we make it impossible for // GetAdjustedTime() to go backward). if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) { - if (state.CorruptionPossible()) { + if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware // problems. @@ -1947,7 +1940,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl for (const auto& tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); } } @@ -1987,11 +1980,19 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl { CAmount txfee = 0; if (!Consensus::CheckTxInputs(tx, state, view, pindex->nHeight, txfee)) { + if (!IsBlockReason(state.GetReason())) { + // CheckTxInputs may return MISSING_INPUTS or + // PREMATURE_SPEND but we can't return that, as it's not + // defined for a block, so we reset the reason flag to + // CONSENSUS here. + state.Invalid(ValidationInvalidReason::CONSENSUS, false, + state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); + } return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } nFees += txfee; if (!MoneyRange(nFees)) { - return state.DoS(100, error("%s: accumulated fee in the block out of range.", __func__), + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: accumulated fee in the block out of range.", __func__), REJECT_INVALID, "bad-txns-accumulated-fee-outofrange"); } @@ -2004,7 +2005,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { - return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: contains a non-BIP68-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); } } @@ -2015,7 +2016,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // * witness (when witness enabled in flags and excludes coinbase) nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) - return state.DoS(100, error("ConnectBlock(): too many sigops"), + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): too many sigops"), REJECT_INVALID, "bad-blk-sigops"); txdata.emplace_back(tx); @@ -2023,9 +2024,20 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ - if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : nullptr)) + if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : nullptr)) { + if (state.GetReason() == ValidationInvalidReason::TX_NOT_STANDARD) { + // CheckInputs may return NOT_STANDARD for extra flags we passed, + // but we can't return that, as it's not defined for a block, so + // we reset the reason flag to CONSENSUS here. + // In the event of a future soft-fork, we may need to + // consider whether rewriting to CONSENSUS or + // RECENT_CONSENSUS_CHANGE would be more appropriate. + state.Invalid(ValidationInvalidReason::CONSENSUS, false, + state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); + } return error("ConnectBlock(): CheckInputs on %s failed with %s", tx.GetHash().ToString(), FormatStateMessage(state)); + } control.Add(vChecks); } @@ -2040,13 +2052,13 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) - return state.DoS(100, + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", block.vtx[0]->GetValueOut(), blockReward), REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) - return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); @@ -2134,8 +2146,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0, true)) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). @@ -2168,8 +2181,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); @@ -2572,7 +2586,7 @@ bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainPar if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. - if (!state.CorruptionPossible()) { + if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { InvalidChainFound(vpindexToConnect.front()); } state = CValidationState(); @@ -2634,6 +2648,14 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { } } +static void LimitValidationInterfaceQueue() { + AssertLockNotHeld(cs_main); + + if (GetMainSignals().CallbacksPending() > 10) { + SyncWithValidationInterfaceQueue(); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either nullptr or a pointer to a block @@ -2662,15 +2684,13 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& do { boost::this_thread::interruption_point(); - if (GetMainSignals().CallbacksPending() > 10) { - // Block until the validation queue drains. This should largely - // never happen in normal operation, however may happen during - // reindex, causing memory blowup if we run too far ahead. - // Note that if a validationinterface callback ends up calling - // ActivateBestChain this may lead to a deadlock! We should - // probably have a DEBUG_LOCKORDER test for this in the future. - SyncWithValidationInterfaceQueue(); - } + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. + LimitValidationInterfaceQueue(); { LOCK(cs_main); @@ -2781,64 +2801,85 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { - AssertLockHeld(cs_main); + CBlockIndex* to_mark_failed = pindex; + bool pindex_was_in_chain = false; + int disconnected = 0; - // We first disconnect backwards and then mark the blocks as invalid. - // This prevents a case where pruned nodes may fail to invalidateblock - // and be left unable to start as they have no tip candidates (as there - // are no blocks that meet the "have data and are not invalid per - // nStatus" criteria for inclusion in setBlockIndexCandidates). + // Disconnect (descendants of) pindex, and mark them invalid. + while (true) { + if (ShutdownRequested()) break; - bool pindex_was_in_chain = false; - CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); - DisconnectedBlockTransactions disconnectpool; - while (chainActive.Contains(pindex)) { + LOCK(cs_main); + if (!chainActive.Contains(pindex)) break; pindex_was_in_chain = true; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams, &disconnectpool)) { - // It's probably hopeless to try to make the mempool consistent - // here if DisconnectTip failed, but we can try. - UpdateMempoolForReorg(disconnectpool, false); - return false; - } - } - - // Now mark the blocks we just disconnected as descendants invalid - // (note this may not be all descendants). - while (pindex_was_in_chain && invalid_walk_tip != pindex) { - invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + DisconnectedBlockTransactions disconnectpool; + bool ret = DisconnectTip(state, chainparams, &disconnectpool); + // DisconnectTip will add transactions to disconnectpool. + // Adjust the mempool to be consistent with the new tip, adding + // transactions back to the mempool if disconnecting was successful, + // and we're not doing a very deep invalidation (in which case + // keeping the mempool up to date is probably futile anyway). + UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + if (!ret) return false; + assert(invalid_walk_tip->pprev == chainActive.Tip()); + + // We immediately mark the disconnected blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(invalid_walk_tip); setBlockIndexCandidates.erase(invalid_walk_tip); - invalid_walk_tip = invalid_walk_tip->pprev; + setBlockIndexCandidates.insert(invalid_walk_tip->pprev); + if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { + // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children + // need to be BLOCK_FAILED_CHILD instead. + to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(to_mark_failed); + } + + // Track the last disconnected block, so we can correct its BLOCK_FAILED_CHILD status in future + // iterations, or, if it's the last one, call InvalidChainFound on it. + to_mark_failed = invalid_walk_tip; } - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); - m_failed_blocks.insert(pindex); + { + LOCK(cs_main); + if (chainActive.Contains(to_mark_failed)) { + // If the to-be-marked invalid block is in the active chain, something is interfering and we can't proceed. + return false; + } - // DisconnectTip will add transactions to disconnectpool; try to add these - // back to the mempool. - UpdateMempoolForReorg(disconnectpool, true); + // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain + to_mark_failed->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(to_mark_failed); + setBlockIndexCandidates.erase(to_mark_failed); + m_failed_blocks.insert(to_mark_failed); - // The resulting new best tip may not be in setBlockIndexCandidates anymore, so - // add it again. - BlockMap::iterator it = mapBlockIndex.begin(); - while (it != mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { - setBlockIndexCandidates.insert(it->second); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so + // add it again. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; } - it++; - } - InvalidChainFound(pindex); + InvalidChainFound(to_mark_failed); + } // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); + uiInterface.NotifyBlockTip(IsInitialBlockDownload(), to_mark_failed->pprev); } return true; } @@ -2922,7 +2963,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2968,7 +3009,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3003,21 +3044,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3025,32 +3058,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int return true; } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; @@ -3060,7 +3084,7 @@ static bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, { // Check proof of work matches claimed amount if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) - return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "high-hash", "proof of work failed"); return true; } @@ -3082,13 +3106,13 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) - return state.DoS(100, false, REJECT_INVALID, "bad-txnmrklroot", true, "hashMerkleRoot mismatch"); + return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txnmrklroot", "hashMerkleRoot mismatch"); // Check for merkle tree malleability (CVE-2012-2459): repeating sequences // of transactions in a block without affecting the merkle root of a block, // while still invalidating it. if (mutated) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-duplicate", true, "duplicate transaction"); + return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txns-duplicate", "duplicate transaction"); } // All potential-corruption validation must be done before we do any @@ -3099,19 +3123,19 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P // Size limits if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-missing", "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i]->IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-multiple", "more than one coinbase"); // Check transactions for (const auto& tx : block.vtx) if (!CheckTransaction(*tx, state, true)) - return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), + return state.Invalid(state.GetReason(), false, state.GetRejectCode(), state.GetRejectReason(), strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), state.GetDebugMessage())); unsigned int nSigOps = 0; @@ -3120,7 +3144,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P nSigOps += GetLegacySigOpCount(*tx); } if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) - return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-sigops", "out-of-bounds SigOpCount"); if (fCheckPOW && fCheckMerkleRoot) block.fChecked = true; @@ -3196,6 +3220,22 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } +//! Returns last CBlockIndex* that is a checkpoint +static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) +{ + const MapCheckpoints& checkpoints = data.mapCheckpoints; + + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) + { + const uint256& hash = i.second; + CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { + return pindex; + } + } + return nullptr; +} + /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). @@ -3213,32 +3253,32 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // Check proof of work const Consensus::Params& consensusParams = params.GetConsensus(); if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) - return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "bad-diffbits", "incorrect proof of work"); // Check against checkpoints if (fCheckpointsEnabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // MapBlockIndex. - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(params.Checkpoints()); + CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); + return state.Invalid(ValidationInvalidReason::BLOCK_CHECKPOINT, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); } // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); // Check timestamp if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) - return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); + return state.Invalid(ValidationInvalidReason::BLOCK_TIME_FUTURE, false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: // check for version 2, 3 and 4 upgrades if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) - return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; @@ -3268,7 +3308,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // Check that all transactions are finalized for (const auto& tx : block.vtx) { if (!IsFinalTx(*tx, nHeight, nLockTimeCutoff)) { - return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-nonfinal", "non-final transaction"); } } @@ -3278,7 +3318,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { - return state.DoS(100, false, REJECT_INVALID, "bad-cb-height", false, "block height mismatch in coinbase"); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-height", "block height mismatch in coinbase"); } } @@ -3300,11 +3340,11 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // already does not permit it, it is impossible to trigger in the // witness tree. if (block.vtx[0]->vin[0].scriptWitness.stack.size() != 1 || block.vtx[0]->vin[0].scriptWitness.stack[0].size() != 32) { - return state.DoS(100, false, REJECT_INVALID, "bad-witness-nonce-size", true, strprintf("%s : invalid witness reserved value size", __func__)); + return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); } CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0]->vin[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); if (memcmp(hashWitness.begin(), &block.vtx[0]->vout[commitpos].scriptPubKey[6], 32)) { - return state.DoS(100, false, REJECT_INVALID, "bad-witness-merkle-match", true, strprintf("%s : witness merkle commitment mismatch", __func__)); + return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; } @@ -3314,7 +3354,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c if (!fHaveWitness) { for (const auto& tx : block.vtx) { if (tx->HasWitness()) { - return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); + return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); } } } @@ -3326,7 +3366,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // the block hash, so we couldn't mark the block as permanently // failed). if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { - return state.DoS(100, false, REJECT_INVALID, "bad-blk-weight", false, strprintf("%s : weight limit failed", __func__)); + return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); } return true; @@ -3346,7 +3386,7 @@ bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); + return state.Invalid(ValidationInvalidReason::CACHED_INVALID, error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); return true; } @@ -3357,10 +3397,10 @@ bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); + return state.Invalid(ValidationInvalidReason::BLOCK_MISSING_PREV, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) - return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); @@ -3397,7 +3437,7 @@ bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& setDirtyBlockIndex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); } } } @@ -3435,26 +3475,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - CDiskBlockPos blockPos; + FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { error("%s: FindBlockPos failed", __func__); - return CDiskBlockPos(); + return FlatFilePos(); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); - return CDiskBlockPos(); + return FlatFilePos(); } } return blockPos; } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3501,7 +3541,8 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali if (!CheckBlock(block, state, chainparams.GetConsensus()) || !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { - if (state.IsInvalid() && !state.CorruptionPossible()) { + assert(IsBlockReason(state.GetReason())); + if (state.IsInvalid() && state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } @@ -3516,7 +3557,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali // Write block to history file if (fNewBlock) *fNewBlock = true; try { - CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3647,9 +3688,9 @@ void PruneOneBlockFile(const int fileNumber) void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3757,52 +3798,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) +static FlatFileSeq BlockFileSeq() { - uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); } -static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +static FlatFileSeq UndoFileSeq() { - if (pos.IsNull()) - return nullptr; - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) - file = fsbridge::fopen(path, "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } - } - return file; + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { + return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) @@ -3909,7 +3926,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -4162,38 +4179,114 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) { return g_chainstate.ReplayBlocks(params, view); } -bool CChainState::RewindBlockIndex(const CChainParams& params) +//! Helper for CChainState::RewindBlockIndex +void CChainState::EraseBlockData(CBlockIndex* index) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(!chainActive.Contains(index)); // Make sure this block isn't active + + // Reduce validity + index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + index->nFile = 0; + index->nDataPos = 0; + index->nUndoPos = 0; + // Remove various other things + index->nTx = 0; + index->nChainTx = 0; + index->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(index); + // Update indexes + setBlockIndexCandidates.erase(index); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(index->pprev); + while (ret.first != ret.second) { + if (ret.first->second == index) { + mapBlocksUnlinked.erase(ret.first++); + } else { + ++ret.first; + } + } + // Mark parent as eligible for main chain again + if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { + setBlockIndexCandidates.insert(index->pprev); + } +} +bool CChainState::RewindBlockIndex(const CChainParams& params) +{ // Note that during -reindex-chainstate we are called with an empty chainActive! - int nHeight = 1; - while (nHeight <= chainActive.Height()) { - // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all - // blocks in ConnectBlock, we don't need to go back and - // re-download/re-verify blocks from before segwit actually activated. - if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; + // First erase all post-segwit blocks without witness not in the main chain, + // as this can we done without costly DisconnectTip calls. Active + // blocks will be dealt with below (releasing cs_main in between). + { + LOCK(cs_main); + for (const auto& entry : mapBlockIndex) { + if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(entry.second)) { + EraseBlockData(entry.second); + } } - nHeight++; } + // Find what height we need to reorganize to. + CBlockIndex *tip; + int nHeight = 1; + { + LOCK(cs_main); + while (nHeight <= chainActive.Height()) { + // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all + // blocks in ConnectBlock, we don't need to go back and + // re-download/re-verify blocks from before segwit actually activated. + if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } + + tip = chainActive.Tip(); + } // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; - CBlockIndex* pindex = chainActive.Tip(); - while (chainActive.Height() >= nHeight) { - if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; - } - if (!DisconnectTip(state, params, nullptr)) { - return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", pindex->nHeight, FormatStateMessage(state)); + // Loop until the tip is below nHeight, or we reach a pruned block. + while (!ShutdownRequested()) { + { + LOCK(cs_main); + // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) + assert(tip == chainActive.Tip()); + if (tip == nullptr || tip->nHeight < nHeight) break; + if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + + // Disconnect block + if (!DisconnectTip(state, params, nullptr)) { + return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, FormatStateMessage(state)); + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + EraseBlockData(tip); + + tip = tip->pprev; } + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); + // Occasionally flush state to disk. if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); @@ -4201,53 +4294,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) } } - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (const auto& entry : mapBlockIndex) { - CBlockIndex* pindexIter = entry.second; - - // Note: If we encounter an insufficiently validated block that - // is on chainActive, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on chainActive at this point - // must not have their validity reduced. - if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { - // Reduce validity - pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - pindexIter->nFile = 0; - pindexIter->nDataPos = 0; - pindexIter->nUndoPos = 0; - // Remove various other things - pindexIter->nTx = 0; - pindexIter->nChainTx = 0; - pindexIter->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(pindexIter); - // Update indexes - setBlockIndexCandidates.erase(pindexIter); - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); - while (ret.first != ret.second) { - if (ret.first->second == pindexIter) { - mapBlocksUnlinked.erase(ret.first++); - } else { - ++ret.first; - } - } - } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->HaveTxsDownloaded()) { - setBlockIndexCandidates.insert(pindexIter); - } - } - - if (chainActive.Tip() != nullptr) { - // We can't prune block index candidates based on our tip if we have - // no tip due to chainActive being empty! - PruneBlockIndexCandidates(); + { + LOCK(cs_main); + if (chainActive.Tip() != nullptr) { + // We can't prune block index candidates based on our tip if we have + // no tip due to chainActive being empty! + PruneBlockIndexCandidates(); - CheckBlockIndex(params.GetConsensus()); + CheckBlockIndex(params.GetConsensus()); + } } return true; @@ -4342,7 +4397,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) try { const CBlock& block = chainparams.GenesisBlock(); - CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); @@ -4359,10 +4414,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return g_chainstate.LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; + static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4448,9 +4503,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { @@ -4697,7 +4752,7 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool() +bool LoadMempool(CTxMemPool& pool) { const CChainParams& chainparams = Params(); int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; @@ -4732,12 +4787,12 @@ bool LoadMempool() CAmount amountdelta = nFeeDelta; if (amountdelta) { - mempool.PrioritiseTransaction(tx->GetHash(), amountdelta); + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } CValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(chainparams, mempool, state, tx, nullptr /* pfMissingInputs */, nTime, + AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nullptr /* pfMissingInputs */, nTime, nullptr /* plTxnReplaced */, false /* bypass_limits */, 0 /* nAbsurdFee */, false /* test_accept */); if (state.IsValid()) { @@ -4747,7 +4802,7 @@ bool LoadMempool() // wallet(s) having loaded it while we were processing // mempool transactions; consider these as valid, instead of // failed, but mark them as 'already there' - if (mempool.exists(tx->GetHash())) { + if (pool.exists(tx->GetHash())) { ++already_there; } else { ++failed; @@ -4763,7 +4818,7 @@ bool LoadMempool() file >> mapDeltas; for (const auto& i : mapDeltas) { - mempool.PrioritiseTransaction(i.first, i.second); + pool.PrioritiseTransaction(i.first, i.second); } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); @@ -4774,7 +4829,7 @@ bool LoadMempool() return true; } -bool DumpMempool() +bool DumpMempool(const CTxMemPool& pool) { int64_t start = GetTimeMicros(); @@ -4785,11 +4840,11 @@ bool DumpMempool() LOCK(dump_mutex); { - LOCK(mempool.cs); - for (const auto &i : mempool.mapDeltas) { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { mapDeltas[i.first] = i.second; } - vinfo = mempool.infoAll(); + vinfo = pool.infoAll(); } int64_t mid = GetTimeMicros(); diff --git a/src/validation.h b/src/validation.h index 1975846b69..7ab6adaf33 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include <versionbits.h> #include <algorithm> +#include <atomic> #include <exception> #include <map> #include <memory> @@ -30,10 +31,9 @@ #include <utility> #include <vector> -#include <atomic> - class CBlockIndex; class CBlockTreeDB; +class CBlockUndo; class CChainParams; class CCoinsViewDB; class CInv; @@ -53,12 +53,6 @@ static const bool DEFAULT_WHITELISTRELAY = true; static const bool DEFAULT_WHITELISTFORCERELAY = false; /** Default for -minrelaytxfee, minimum relay fee for transactions */ static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; -//! -maxtxfee default -static const CAmount DEFAULT_TRANSACTION_MAXFEE = COIN / 10; -//! Discourage users to set fees higher than this amount (in satoshis) per kB -static const CAmount HIGH_TX_FEE_PER_KB = COIN / 100; -//! -maxtxfee will warn if called with a higher fee than this amount (in satoshis) -static const CAmount HIGH_MAX_TX_FEE = 100 * HIGH_TX_FEE_PER_KB; /** Default for -limitancestorcount, max number of in-mempool ancestors */ static const unsigned int DEFAULT_ANCESTOR_LIMIT = 25; /** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */ @@ -114,10 +108,9 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; /** Maximum age of our tip in seconds for us to be considered current for fee estimation */ static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; -/** Default for -permitbaremultisig */ -static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; @@ -149,25 +142,20 @@ extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; -extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap& mapBlockIndex GUARDED_BY(cs_main); -extern const std::string strMessageMagic; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; extern int nScriptCheckThreads; -extern bool fIsBareMultisigStd; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; extern size_t nCoinCacheUsage; /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */ extern CFeeRate minRelayTxFee; -/** Absolute maximum transaction fee (in satoshis) used by wallet and mempool (rejects high fee in sendrawtransaction) */ -extern CAmount maxTxFee; /** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ extern int64_t nMaxTipAge; extern bool fEnableReplacement; @@ -181,9 +169,6 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Minimum disk space required - used in CheckDiskSpace() */ -static const uint64_t nMinDiskSpace = 52428800; - /** Pruning-related variables and constants */ /** True if any block files have ever been pruned. */ extern bool fHavePruned; @@ -245,14 +230,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); -/** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, @@ -263,7 +246,7 @@ bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_m /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ -void ThreadScriptCheck(); +void ThreadScriptCheck(int worker_num); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ @@ -306,9 +289,6 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state); - /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); @@ -391,11 +371,13 @@ void InitScriptExecutionCache(); /** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ @@ -448,7 +430,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -491,10 +473,10 @@ static const unsigned int REJECT_HIGHFEE = 0x100; CBlockFileInfo* GetBlockFileInfo(size_t n); /** Dump the mempool to disk. */ -bool DumpMempool(); +bool DumpMempool(const CTxMemPool& pool); /** Load the mempool from disk. */ -bool LoadMempool(); +bool LoadMempool(CTxMemPool& pool); //! Check whether the block associated with this index entry is pruned or not. inline bool IsBlockPruned(const CBlockIndex* pblockindex) diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 70c274d20e..5d0ee1d1fc 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -25,7 +25,6 @@ struct ValidationInterfaceConnections { boost::signals2::scoped_connection BlockDisconnected; boost::signals2::scoped_connection TransactionRemovedFromMempool; boost::signals2::scoped_connection ChainStateFlushed; - boost::signals2::scoped_connection Broadcast; boost::signals2::scoped_connection BlockChecked; boost::signals2::scoped_connection NewPoWValidBlock; }; @@ -37,7 +36,6 @@ struct MainSignalsInstance { boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; - boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; @@ -101,7 +99,6 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1)); conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); - conns.Broadcast = g_signals.m_internals->Broadcast.connect(std::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.NewPoWValidBlock = g_signals.m_internals->NewPoWValidBlock.connect(std::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, std::placeholders::_1, std::placeholders::_2)); } @@ -175,10 +172,6 @@ void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { }); } -void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { - m_internals->Broadcast(nBestBlockTime, connman); -} - void CMainSignals::BlockChecked(const CBlock& block, const CValidationState& state) { m_internals->BlockChecked(block, state); } diff --git a/src/validationinterface.h b/src/validationinterface.h index f0374e8e78..ea1b2e7e76 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -18,7 +18,6 @@ class CBlockIndex; struct CBlockLocator; class CBlockIndex; class CConnman; -class CReserveScript; class CValidationInterface; class CValidationState; class uint256; @@ -134,8 +133,6 @@ protected: * Called on a background thread. */ virtual void ChainStateFlushed(const CBlockLocator &locator) {} - /** Tells listeners to broadcast their data. */ - virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} /** * Notifies listeners of a block validation result. * If the provided CValidationState IsValid, the provided block @@ -184,7 +181,6 @@ public: void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); void BlockDisconnected(const std::shared_ptr<const CBlock> &); void ChainStateFlushed(const CBlockLocator &); - void Broadcast(int64_t nBestBlockTime, CConnman* connman); void BlockChecked(const CBlock&, const CValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 48a924abfb..9257b272bc 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -36,6 +36,8 @@ public: bool m_avoid_partial_spends; //! Fee estimation mode to control arguments to estimateSmartFee FeeEstimateMode m_fee_mode; + //! Minimum chain depth value for coin availability + int m_min_depth{0}; CCoinControl() { diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 99d880daa0..6a326bfd97 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -84,6 +84,14 @@ bool IsWalletLoaded(const fs::path& wallet_path) return database && database->IsDatabaseLoaded(database_filename); } +fs::path WalletDataFilePath(const fs::path& wallet_path) +{ + fs::path env_directory; + std::string database_filename; + SplitWalletPath(wallet_path, env_directory, database_filename); + return env_directory / database_filename; +} + /** * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. diff --git a/src/wallet/db.h b/src/wallet/db.h index 9df965305a..762fb83a2f 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -101,6 +101,9 @@ public: /** Return whether a wallet database is currently loaded. */ bool IsWalletLoaded(const fs::path& wallet_path); +/** Given a wallet directory path or legacy file path, return path to main data file in the wallet database. */ +fs::path WalletDataFilePath(const fs::path& wallet_path); + /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 7a71aea715..78db4df5e5 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -14,7 +14,9 @@ #include <validation.h> //for mempool access #include <txmempool.h> #include <util/moneystr.h> +#include <util/rbf.h> #include <util/system.h> +#include <util/validation.h> #include <net.h> //! Check whether transaction has descendant in wallet or mempool, or has been @@ -27,9 +29,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } { - LOCK(mempool.cs); - auto it_mp = mempool.mapTx.find(wtx.GetHash()); - if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + if (wallet->chain().hasDescendantsInMempool(wtx.GetHash())) { errors.push_back("Transaction has descendants in the mempool"); return feebumper::Result::INVALID_PARAMETER; } @@ -75,9 +75,11 @@ bool TransactionCanBeBumped(const CWallet* wallet, const uint256& txid) return res == feebumper::Result::OK; } -Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, CAmount total_fee, std::vector<std::string>& errors, - CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) +Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, CAmount total_fee, std::vector<std::string>& errors, + CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) { + new_fee = total_fee; + auto locked_chain = wallet->chain().lock(); LOCK(wallet->cs_wallet); errors.clear(); @@ -121,49 +123,33 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // calculate the old fee and fee-rate old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); CFeeRate nOldFeeRate(old_fee, txSize); - CFeeRate nNewFeeRate; // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay // fee that our node may not be aware of. + CFeeRate nodeIncrementalRelayFee = wallet->chain().relayIncrementalFee(); CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); - if (::incrementalRelayFee > walletIncrementalRelayFee) { - walletIncrementalRelayFee = ::incrementalRelayFee; + if (nodeIncrementalRelayFee > walletIncrementalRelayFee) { + walletIncrementalRelayFee = nodeIncrementalRelayFee; } - if (total_fee > 0) { - CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (total_fee < minTotalFee) { - errors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", - FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - return Result::INVALID_PARAMETER; - } - CAmount requiredFee = GetRequiredFee(*wallet, maxNewTxSize); - if (total_fee < requiredFee) { - errors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", - FormatMoney(requiredFee))); - return Result::INVALID_PARAMETER; - } - new_fee = total_fee; - nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); - } else { - new_fee = GetMinimumFee(*wallet, maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); - nNewFeeRate = CFeeRate(new_fee, maxNewTxSize); - - // New fee rate must be at least old rate + minimum incremental relay rate - // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized - // in that unit (fee per kb). - // However, nOldFeeRate is a calculated value from the tx fee/size, so - // add 1 satoshi to the result, because it may have been rounded down. - if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { - nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - new_fee = nNewFeeRate.GetFee(maxNewTxSize); - } + CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + nodeIncrementalRelayFee.GetFee(maxNewTxSize); + if (total_fee < minTotalFee) { + errors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(nodeIncrementalRelayFee.GetFee(maxNewTxSize)))); + return Result::INVALID_PARAMETER; + } + CAmount requiredFee = GetRequiredFee(*wallet, maxNewTxSize); + if (total_fee < requiredFee) { + errors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + FormatMoney(requiredFee))); + return Result::INVALID_PARAMETER; } // Check that in all cases the new fee doesn't violate maxTxFee - if (new_fee > maxTxFee) { - errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(new_fee), FormatMoney(maxTxFee))); + const CAmount max_tx_fee = wallet->m_default_max_tx_fee; + if (new_fee > max_tx_fee) { + errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)", + FormatMoney(new_fee), FormatMoney(max_tx_fee))); return Result::WALLET_ERROR; } @@ -172,15 +158,15 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a // moment earlier. In this case, we report an error to the user, who may use total_fee to make an adjustment. - CFeeRate minMempoolFeeRate = mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + CFeeRate minMempoolFeeRate = wallet->chain().mempoolMinFee(); + CFeeRate nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { errors.push_back(strprintf( "New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- " - "the totalFee value should be at least %s or the settxfee value should be at least %s to add transaction", + "the totalFee value should be at least %s to add transaction", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), - FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), - FormatMoney(minMempoolFeeRate.GetFeePerK()))); + FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)))); return Result::WALLET_ERROR; } @@ -197,7 +183,7 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; - if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet, ::feeEstimator))) { + if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet))) { wallet->WalletLogPrintf("Bumping fee and discarding dust output\n"); new_fee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); @@ -210,6 +196,109 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin } } + return Result::OK; +} + + +Result CreateRateBumpTransaction(CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<std::string>& errors, + CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) +{ + // We are going to modify coin control later, copy to re-use + CCoinControl new_coin_control(coin_control); + + auto locked_chain = wallet->chain().lock(); + LOCK(wallet->cs_wallet); + errors.clear(); + auto it = wallet->mapWallet.find(txid); + if (it == wallet->mapWallet.end()) { + errors.push_back("Invalid or non-wallet transaction id"); + return Result::INVALID_ADDRESS_OR_KEY; + } + const CWalletTx& wtx = it->second; + + Result result = PreconditionChecks(*locked_chain, wallet, wtx, errors); + if (result != Result::OK) { + return result; + } + + // Fill in recipients(and preserve a single change key if there is one) + std::vector<CRecipient> recipients; + for (const auto& output : wtx.tx->vout) { + if (!wallet->IsChange(output)) { + CRecipient recipient = {output.scriptPubKey, output.nValue, false}; + recipients.push_back(recipient); + } else { + CTxDestination change_dest; + ExtractDestination(output.scriptPubKey, change_dest); + new_coin_control.destChange = change_dest; + } + } + + // Get the fee rate of the original transaction. This is calculated from + // the tx fee/vsize, so it may have been rounded down. Add 1 satoshi to the + // result. + old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); + // Feerate of thing we are bumping + CFeeRate feerate(old_fee, txSize); + feerate += CFeeRate(1); + + // The node has a configurable incremental relay fee. Increment the fee by + // the minimum of that and the wallet's conservative + // WALLET_INCREMENTAL_RELAY_FEE value to future proof against changes to + // network wide policy for incremental relay fee that our node may not be + // aware of. This ensures we're over the over the required relay fee rate + // (BIP 125 rule 4). The replacement tx will be at least as large as the + // original tx, so the total fee will be greater (BIP 125 rule 3) + CFeeRate node_incremental_relay_fee = wallet->chain().relayIncrementalFee(); + CFeeRate wallet_incremental_relay_fee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); + feerate += std::max(node_incremental_relay_fee, wallet_incremental_relay_fee); + + // Fee rate must also be at least the wallet's GetMinimumFeeRate + CFeeRate min_feerate(GetMinimumFeeRate(*wallet, new_coin_control, /* feeCalc */ nullptr)); + + // Set the required fee rate for the replacement transaction in coin control. + new_coin_control.m_feerate = std::max(feerate, min_feerate); + + // Fill in required inputs we are double-spending(all of them) + // N.B.: bip125 doesn't require all the inputs in the replaced transaction to be + // used in the replacement transaction, but it's very important for wallets to make + // sure that happens. If not, it would be possible to bump a transaction A twice to + // A2 and A3 where A2 and A3 don't conflict (or alternatively bump A to A2 and A2 + // to A3 where A and A3 don't conflict). If both later get confirmed then the sender + // has accidentally double paid. + for (const auto& inputs : wtx.tx->vin) { + new_coin_control.Select(COutPoint(inputs.prevout)); + } + new_coin_control.fAllowOtherInputs = true; + + // We cannot source new unconfirmed inputs(bip125 rule 2) + new_coin_control.m_min_depth = 1; + + CTransactionRef tx_new = MakeTransactionRef(); + CReserveKey reservekey(wallet); + CAmount fee_ret; + int change_pos_in_out = -1; // No requested location for change + std::string fail_reason; + if (!wallet->CreateTransaction(*locked_chain, recipients, tx_new, reservekey, fee_ret, change_pos_in_out, fail_reason, new_coin_control, false)) { + errors.push_back("Unable to create transaction: " + fail_reason); + return Result::WALLET_ERROR; + } + + // If change key hasn't been ReturnKey'ed by this point, we take it out of keypool + reservekey.KeepKey(); + + // Write back new fee if successful + new_fee = fee_ret; + + // Write back transaction + mtx = CMutableTransaction(*tx_new); + // Mark new tx not replaceable, if requested. + if (!coin_control.m_signal_bip125_rbf.get_value_or(wallet->m_signal_rbf)) { + for (auto& input : mtx.vin) { + if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; + } + } return Result::OK; } @@ -247,7 +336,7 @@ Result CommitTransaction(CWallet* wallet, const uint256& txid, CMutableTransacti CReserveKey reservekey(wallet); CValidationState state; - if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, g_connman.get(), state)) { + if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, state)) { // NOTE: CommitTransaction never returns false, so this should never happen. errors.push_back(strprintf("The transaction was rejected: %s", FormatStateMessage(state))); return Result::WALLET_ERROR; diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index a1eb5d0e0d..f9cbfc5f68 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -28,8 +28,8 @@ enum class Result //! Return whether transaction can be bumped. bool TransactionCanBeBumped(const CWallet* wallet, const uint256& txid); -//! Create bumpfee transaction. -Result CreateTransaction(const CWallet* wallet, +//! Create bumpfee transaction based on total amount. +Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, CAmount total_fee, @@ -38,6 +38,15 @@ Result CreateTransaction(const CWallet* wallet, CAmount& new_fee, CMutableTransaction& mtx); +//! Create bumpfee transaction based on feerate estimates. +Result CreateRateBumpTransaction(CWallet* wallet, + const uint256& txid, + const CCoinControl& coin_control, + std::vector<std::string>& errors, + CAmount& old_fee, + CAmount& new_fee, + CMutableTransaction& mtx); + //! Sign the new transaction, //! @return false if the tx couldn't be found or if it was //! impossible to create the signature(s) diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 9e2984ff05..d9ae18ed60 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -6,7 +6,6 @@ #include <wallet/fees.h> #include <policy/policy.h> -#include <txmempool.h> #include <util/system.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -19,12 +18,13 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes) } -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc) { - CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, pool, estimator, feeCalc).GetFee(nTxBytes); + CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, feeCalc).GetFee(nTxBytes); // Always obey the maximum - if (fee_needed > maxTxFee) { - fee_needed = maxTxFee; + const CAmount max_tx_fee = wallet.m_default_max_tx_fee; + if (fee_needed > max_tx_fee) { + fee_needed = max_tx_fee; if (feeCalc) feeCalc->reason = FeeReason::MAXTXFEE; } return fee_needed; @@ -32,10 +32,10 @@ CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinC CFeeRate GetRequiredFeeRate(const CWallet& wallet) { - return std::max(wallet.m_min_fee, ::minRelayTxFee); + return std::max(wallet.m_min_fee, wallet.chain().relayMinFee()); } -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc) { /* User control of how to calculate fee uses the following parameter precedence: 1. coin_control.m_feerate @@ -64,7 +64,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (coin_control.m_fee_mode == FeeEstimateMode::CONSERVATIVE) conservative_estimate = true; else if (coin_control.m_fee_mode == FeeEstimateMode::ECONOMICAL) conservative_estimate = false; - feerate_needed = estimator.estimateSmartFee(target, feeCalc, conservative_estimate); + feerate_needed = wallet.chain().estimateSmartFee(target, conservative_estimate, feeCalc); if (feerate_needed == CFeeRate(0)) { // if we don't have enough data for estimateSmartFee, then use fallback fee feerate_needed = wallet.m_fallback_fee; @@ -74,7 +74,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (wallet.m_fallback_fee == CFeeRate(0)) return feerate_needed; } // Obey mempool min fee when using smart fee estimation - CFeeRate min_mempool_feerate = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + CFeeRate min_mempool_feerate = wallet.chain().mempoolMinFee(); if (feerate_needed < min_mempool_feerate) { feerate_needed = min_mempool_feerate; if (feeCalc) feeCalc->reason = FeeReason::MEMPOOL_MIN; @@ -90,13 +90,13 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr return feerate_needed; } -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator) +CFeeRate GetDiscardRate(const CWallet& wallet) { - unsigned int highest_target = estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - CFeeRate discard_rate = estimator.estimateSmartFee(highest_target, nullptr /* FeeCalculation */, false /* conservative */); + unsigned int highest_target = wallet.chain().estimateMaxBlocks(); + CFeeRate discard_rate = wallet.chain().estimateSmartFee(highest_target, false /* conservative */); // Don't let discard_rate be greater than longest possible fee estimate if we get a valid fee estimate discard_rate = (discard_rate == CFeeRate(0)) ? wallet.m_discard_rate : std::min(discard_rate, wallet.m_discard_rate); // Discard rate must be at least dustRelayFee - discard_rate = std::max(discard_rate, ::dustRelayFee); + discard_rate = std::max(discard_rate, wallet.chain().relayDustFee()); return discard_rate; } diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 6bfee456c0..434f211dc2 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -8,10 +8,8 @@ #include <amount.h> -class CBlockPolicyEstimator; class CCoinControl; class CFeeRate; -class CTxMemPool; class CWallet; struct FeeCalculation; @@ -25,7 +23,7 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes); * Estimate the minimum fee considering user set parameters * and the required fee */ -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the minimum required feerate taking into account the @@ -37,11 +35,11 @@ CFeeRate GetRequiredFeeRate(const CWallet& wallet); * Estimate the minimum fee rate considering user set parameters * and the required fee */ -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the maximum feerate for discarding change. */ -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator); +CFeeRate GetDiscardRate(const CWallet& wallet); #endif // BITCOIN_WALLET_FEES_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 20d540c8db..47ef01bfd1 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -9,6 +9,7 @@ #include <net.h> #include <scheduler.h> #include <outputtype.h> +#include <util/error.h> #include <util/system.h> #include <util/moneystr.h> #include <validation.h> @@ -47,6 +48,8 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), false, OptionsCategory::WALLET); gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), false, OptionsCategory::WALLET); + gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", + CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), false, OptionsCategory::WALLET); gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)", @@ -123,58 +126,6 @@ bool WalletInit::ParameterInteraction() const if (gArgs.GetArg("-prune", 0) && gArgs.GetBoolArg("-rescan", false)) return InitError(_("Rescans are not possible in pruned mode. You will need to use -reindex which will download the whole blockchain again.")); - if (::minRelayTxFee.GetFeePerK() > HIGH_TX_FEE_PER_KB) - InitWarning(AmountHighWarn("-minrelaytxfee") + " " + - _("The wallet will avoid paying less than the minimum relay fee.")); - - return true; -} - -bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) -{ - if (gArgs.IsArgSet("-walletdir")) { - fs::path wallet_dir = gArgs.GetArg("-walletdir", ""); - boost::system::error_code error; - // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory - fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); - if (error || !fs::exists(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); - } else if (!fs::is_directory(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); - // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version - } else if (!wallet_dir.is_absolute()) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); - } - gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); - } - - LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); - - uiInterface.InitMessage(_("Verifying wallet(s)...")); - - // Parameter interaction code should have thrown an error if -salvagewallet - // was enabled with more than wallet file, so the wallet_files size check - // here should have no effect. - bool salvage_wallet = gArgs.GetBoolArg("-salvagewallet", false) && wallet_files.size() <= 1; - - // Keep track of each wallet absolute path to detect duplicates. - std::set<fs::path> wallet_paths; - - for (const auto& wallet_file : wallet_files) { - WalletLocation location(wallet_file); - - if (!wallet_paths.insert(location.GetPath()).second) { - return InitError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); - } - - std::string error_string; - std::string warning_string; - bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warning_string); - if (!error_string.empty()) InitError(error_string); - if (!warning_string.empty()) InitWarning(warning_string); - if (!verify_success) return false; - } - return true; } @@ -187,51 +138,3 @@ void WalletInit::Construct(InitInterfaces& interfaces) const gArgs.SoftSetArg("-wallet", ""); interfaces.chain_clients.emplace_back(interfaces::MakeWalletClient(*interfaces.chain, gArgs.GetArgs("-wallet"))); } - -bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) -{ - for (const std::string& walletFile : wallet_files) { - std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile)); - if (!pwallet) { - return false; - } - AddWallet(pwallet); - } - - return true; -} - -void StartWallets(CScheduler& scheduler) -{ - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - pwallet->postInitProcess(); - } - - // Run a thread to flush wallet periodically - scheduler.scheduleEvery(MaybeCompactWalletDB, 500); -} - -void FlushWallets() -{ - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - pwallet->Flush(false); - } -} - -void StopWallets() -{ - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - pwallet->Flush(true); - } -} - -void UnloadWallets() -{ - auto wallets = GetWallets(); - while (!wallets.empty()) { - auto wallet = wallets.back(); - wallets.pop_back(); - RemoveWallet(wallet); - UnloadWallet(std::move(wallet)); - } -} diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp new file mode 100644 index 0000000000..79c5f439df --- /dev/null +++ b/src/wallet/load.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/load.h> + +#include <interfaces/chain.h> +#include <scheduler.h> +#include <util/system.h> +#include <wallet/wallet.h> + +bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) +{ + if (gArgs.IsArgSet("-walletdir")) { + fs::path wallet_dir = gArgs.GetArg("-walletdir", ""); + boost::system::error_code error; + // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory + fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); + if (error || !fs::exists(wallet_dir)) { + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + return false; + } else if (!fs::is_directory(wallet_dir)) { + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + return false; + // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version + } else if (!wallet_dir.is_absolute()) { + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + return false; + } + gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); + } + + LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); + + chain.initMessage(_("Verifying wallet(s)...")); + + // Parameter interaction code should have thrown an error if -salvagewallet + // was enabled with more than wallet file, so the wallet_files size check + // here should have no effect. + bool salvage_wallet = gArgs.GetBoolArg("-salvagewallet", false) && wallet_files.size() <= 1; + + // Keep track of each wallet absolute path to detect duplicates. + std::set<fs::path> wallet_paths; + + for (const auto& wallet_file : wallet_files) { + WalletLocation location(wallet_file); + + if (!wallet_paths.insert(location.GetPath()).second) { + chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); + return false; + } + + std::string error_string; + std::string warning_string; + bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warning_string); + if (!error_string.empty()) chain.initError(error_string); + if (!warning_string.empty()) chain.initWarning(warning_string); + if (!verify_success) return false; + } + + return true; +} + +bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) +{ + for (const std::string& walletFile : wallet_files) { + std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile)); + if (!pwallet) { + return false; + } + AddWallet(pwallet); + } + + return true; +} + +void StartWallets(CScheduler& scheduler) +{ + for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + pwallet->postInitProcess(); + } + + // Schedule periodic wallet flushes and tx rebroadcasts + scheduler.scheduleEvery(MaybeCompactWalletDB, 500); + scheduler.scheduleEvery(MaybeResendWalletTxs, 1000); +} + +void FlushWallets() +{ + for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + pwallet->Flush(false); + } +} + +void StopWallets() +{ + for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + pwallet->Flush(true); + } +} + +void UnloadWallets() +{ + auto wallets = GetWallets(); + while (!wallets.empty()) { + auto wallet = wallets.back(); + wallets.pop_back(); + RemoveWallet(wallet); + UnloadWallet(std::move(wallet)); + } +} diff --git a/src/wallet/load.h b/src/wallet/load.h new file mode 100644 index 0000000000..9bb6f2166e --- /dev/null +++ b/src/wallet/load.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_LOAD_H +#define BITCOIN_WALLET_LOAD_H + +#include <string> +#include <vector> + +class CScheduler; + +namespace interfaces { +class Chain; +} // namespace interfaces + +//! Responsible for reading and validating the -wallet arguments and verifying the wallet database. +//! This function will perform salvage on the wallet if requested, as long as only one wallet is +//! being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). +bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); + +//! Load wallet databases. +bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); + +//! Complete startup of wallets. +void StartWallets(CScheduler& scheduler); + +//! Flush all wallets in preparation for shutdown. +void FlushWallets(); + +//! Stop all wallets. Wallets will be flushed first. +void StopWallets(); + +//! Close all wallets. +void UnloadWallets(); + +#endif // BITCOIN_WALLET_LOAD_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 5b0592e06d..c339e111ba 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -157,8 +157,9 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); @@ -313,8 +314,9 @@ UniValue importaddress(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -346,7 +348,11 @@ UniValue importaddress(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -501,8 +507,9 @@ UniValue importpubkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -529,7 +536,11 @@ UniValue importpubkey(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -562,8 +573,9 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.ToString()); - if (fPruneMode) + if (pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (!reserver.reserve()) { @@ -591,11 +603,11 @@ UniValue importwallet(const JSONRPCRequest& request) // Use uiInterface.ShowProgress instead of pwallet.ShowProgress because pwallet.ShowProgress has a cancel button tied to AbortRescan which // we don't want for this progress bar showing the import progress. uiInterface.ShowProgress does not have a cancel button. - uiInterface.ShowProgress(strprintf("%s " + _("Importing..."), pwallet->GetDisplayName()), 0, false); // show progress dialog in GUI + pwallet->chain().showProgress(strprintf("%s " + _("Importing..."), pwallet->GetDisplayName()), 0, false); // show progress dialog in GUI std::vector<std::tuple<CKey, int64_t, bool, std::string>> keys; std::vector<std::pair<CScript, int64_t>> scripts; while (file.good()) { - uiInterface.ShowProgress("", std::max(1, std::min(50, (int)(((double)file.tellg() / (double)nFilesize) * 100))), false); + pwallet->chain().showProgress("", std::max(1, std::min(50, (int)(((double)file.tellg() / (double)nFilesize) * 100))), false); std::string line; std::getline(file, line); if (line.empty() || line[0] == '#') @@ -633,13 +645,13 @@ UniValue importwallet(const JSONRPCRequest& request) file.close(); // We now know whether we are importing private keys, so we can error if private keys are disabled if (keys.size() > 0 && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when private keys are disabled"); } double total = (double)(keys.size() + scripts.size()); double progress = 0; for (const auto& key_tuple : keys) { - uiInterface.ShowProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); + pwallet->chain().showProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); const CKey& key = std::get<0>(key_tuple); int64_t time = std::get<1>(key_tuple); bool has_label = std::get<2>(key_tuple); @@ -664,7 +676,7 @@ UniValue importwallet(const JSONRPCRequest& request) progress++; } for (const auto& script_pair : scripts) { - uiInterface.ShowProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); + pwallet->chain().showProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); const CScript& script = script_pair.first; int64_t time = script_pair.second; CScriptID id(script); @@ -683,10 +695,10 @@ UniValue importwallet(const JSONRPCRequest& request) } progress++; } - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI pwallet->UpdateTimeFirstKey(nTimeBegin); } - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI RescanWallet(*pwallet, reserver, nTimeBegin, false /* update */); pwallet->MarkDirty(); @@ -888,7 +900,7 @@ struct ImportData // Output data std::set<CScript> import_scripts; std::map<CKeyID, bool> used_keys; //!< Import these private keys if available (the value indicates whether if the key is required for solvability) - std::map<CKeyID, KeyOriginInfo> key_origins; + std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> key_origins; }; enum class ScriptContext @@ -1185,6 +1197,9 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID bool spendable = std::all_of(pubkey_map.begin(), pubkey_map.end(), [&](const std::pair<CKeyID, CPubKey>& used_key) { return privkey_map.count(used_key.first) > 0; + }) && std::all_of(import_data.key_origins.begin(), import_data.key_origins.end(), + [&](const std::pair<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& entry) { + return privkey_map.count(entry.first) > 0; }); if (!watch_only && !spendable) { warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); @@ -1262,7 +1277,10 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } pwallet->UpdateTimeFirstKey(timestamp); - } + } + for (const auto& entry : import_data.key_origins) { + pwallet->AddKeyOrigin(entry.second.first, entry.second.second); + } for (const CKeyID& id : ordered_pubkeys) { auto entry = pubkey_map.find(id); if (entry == pubkey_map.end()) { @@ -1273,10 +1291,6 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } - const auto& key_orig_it = import_data.key_origins.find(id); - if (key_orig_it != import_data.key_origins.end()) { - pwallet->AddKeyOrigin(pubkey, key_orig_it->second); - } pwallet->mapKeyMetadata[id].nCreateTime = timestamp; // Add to keypool only works with pubkeys @@ -1464,7 +1478,11 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 97c6c38be1..5a22508c4b 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -7,7 +7,6 @@ #include <chain.h> #include <consensus/validation.h> #include <core_io.h> -#include <httpserver.h> #include <init.h> #include <interfaces/chain.h> #include <validation.h> @@ -19,8 +18,7 @@ #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> -#include <rpc/mining.h> -#include <rpc/rawtransaction.h> +#include <rpc/rawtransaction_util.h> #include <rpc/server.h> #include <rpc/util.h> #include <script/descriptor.h> @@ -28,8 +26,11 @@ #include <shutdown.h> #include <timedata.h> #include <util/bip32.h> +#include <util/fees.h> #include <util/system.h> #include <util/moneystr.h> +#include <util/url.h> +#include <util/validation.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> #include <wallet/psbtwallet.h> @@ -124,8 +125,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo // Add opt-in RBF status std::string rbfStatus = "no"; if (confirms <= 0) { - LOCK(mempool.cs); - RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); + RBFTransactionState rbfState = chain.isRBFOptIn(*wtx.tx); if (rbfState == RBFTransactionState::UNKNOWN) rbfStatus = "unknown"; else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) @@ -310,7 +310,7 @@ static UniValue setlabel(const JSONRPCRequest& request) static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) { - CAmount curBalance = pwallet->GetBalance(); + CAmount curBalance = pwallet->GetBalance().m_mine_trusted; // Check amount if (nValue <= 0) @@ -319,7 +319,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet if (nValue > curBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -341,7 +341,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet throw JSONRPCError(RPC_WALLET_ERROR, strError); } CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, state)) { strError = strprintf("Error: The transaction was rejected! Reason given: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -425,7 +425,7 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -607,7 +607,6 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -630,8 +629,9 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) @@ -679,7 +679,6 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -696,8 +695,9 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) { @@ -763,12 +763,14 @@ static UniValue getbalance(const JSONRPCRequest& request) min_depth = request.params[1].get_int(); } - isminefilter filter = ISMINE_SPENDABLE; + bool include_watchonly = false; if (!request.params[2].isNull() && request.params[2].get_bool()) { - filter = filter | ISMINE_WATCH_ONLY; + include_watchonly = true; } - return ValueFromAmount(pwallet->GetBalance(filter, min_depth)); + const auto bal = pwallet->GetBalance(min_depth); + + return ValueFromAmount(bal.m_mine_trusted + (include_watchonly ? bal.m_watchonly_trusted : 0)); } static UniValue getunconfirmedbalance(const JSONRPCRequest &request) @@ -796,7 +798,7 @@ static UniValue getunconfirmedbalance(const JSONRPCRequest &request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(pwallet->GetUnconfirmedBalance()); + return ValueFromAmount(pwallet->GetBalance().m_mine_untrusted_pending); } @@ -809,9 +811,7 @@ static UniValue sendmany(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 2 || request.params.size() > 8) - throw std::runtime_error( - RPCHelpMan{"sendmany", + const RPCHelpMan help{"sendmany", "\nSend multiple times. Amounts are double-precision floating point numbers." + HelpRequiringPassphrase(pwallet) + "\n", { @@ -821,7 +821,7 @@ static UniValue sendmany(const JSONRPCRequest& request) {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, }, }, - {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only use the balance confirmed at least this many times."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "Ignored dummy value"}, {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment"}, {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array with addresses.\n" " The fee will be equally deducted from the amount of each selected address.\n" @@ -852,7 +852,11 @@ static UniValue sendmany(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendmany", "\"\", {\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\":0.01,\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\":0.02}, 6, \"testing\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -861,7 +865,7 @@ static UniValue sendmany(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -869,9 +873,6 @@ static UniValue sendmany(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Dummy value must be set to \"\""); } UniValue sendTo = request.params[1].get_obj(); - int nMinDepth = 1; - if (!request.params[2].isNull()) - nMinDepth = request.params[2].get_int(); mapValue_t mapValue; if (!request.params[3].isNull() && !request.params[3].get_str().empty()) @@ -887,7 +888,7 @@ static UniValue sendmany(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -899,7 +900,6 @@ static UniValue sendmany(const JSONRPCRequest& request) std::set<CTxDestination> destinations; std::vector<CRecipient> vecSend; - CAmount totalAmount = 0; std::vector<std::string> keys = sendTo.getKeys(); for (const std::string& name_ : keys) { CTxDestination dest = DecodeDestination(name_); @@ -916,7 +916,6 @@ static UniValue sendmany(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(sendTo[name_]); if (nAmount <= 0) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - totalAmount += nAmount; bool fSubtractFeeFromAmount = false; for (unsigned int idx = 0; idx < subtractFeeFromAmount.size(); idx++) { @@ -931,11 +930,6 @@ static UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - // Check funds - if (totalAmount > pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth)) { - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Wallet has insufficient funds"); - } - // Shuffle recipient list std::shuffle(vecSend.begin(), vecSend.end(), FastRandomContext()); @@ -949,7 +943,7 @@ static UniValue sendmany(const JSONRPCRequest& request) if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, state)) { strFailReason = strprintf("Transaction commit failed:: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); } @@ -1051,8 +1045,6 @@ struct tallyitem static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Minimum confirmations int nMinDepth = 1; if (!params[0].isNull()) @@ -1083,8 +1075,9 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx)) { continue; + } int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < nMinDepth) @@ -1763,7 +1756,7 @@ static UniValue gettransaction(const JSONRPCRequest& request) ListTransactions(*locked_chain, pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); entry.pushKV("details", details); - std::string strHex = EncodeHexTx(*wtx.tx, RPCSerializationFlags()); + std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); entry.pushKV("hex", strHex); return entry; @@ -1981,7 +1974,7 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) // wallet before the following callback is called. If a valid shared pointer // is acquired in the callback then the wallet is still loaded. std::weak_ptr<CWallet> weak_wallet = wallet; - RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet] { + pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet] { if (auto shared_wallet = weak_wallet.lock()) { LOCK(shared_wallet->cs_wallet); shared_wallet->Lock(); @@ -2370,8 +2363,8 @@ static UniValue settxfee(const JSONRPCRequest& request) CFeeRate tx_fee_rate(nAmount, 1000); if (tx_fee_rate == 0) { // automatic selection - } else if (tx_fee_rate < ::minRelayTxFee) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", ::minRelayTxFee.ToString())); + } else if (tx_fee_rate < pwallet->chain().relayMinFee()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", pwallet->chain().relayMinFee().ToString())); } else if (tx_fee_rate < pwallet->m_min_fee) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than wallet min fee (%s)", pwallet->m_min_fee.ToString())); } @@ -2427,11 +2420,12 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); + const auto bal = pwallet->GetBalance(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); - obj.pushKV("balance", ValueFromAmount(pwallet->GetBalance())); - obj.pushKV("unconfirmed_balance", ValueFromAmount(pwallet->GetUnconfirmedBalance())); - obj.pushKV("immature_balance", ValueFromAmount(pwallet->GetImmatureBalance())); + obj.pushKV("balance", ValueFromAmount(bal.m_mine_trusted)); + obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); + obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); @@ -2671,49 +2665,6 @@ static UniValue unloadwallet(const JSONRPCRequest& request) return NullUniValue; } -static UniValue resendwallettransactions(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( - RPCHelpMan{"resendwallettransactions", - "Immediately re-broadcast unconfirmed wallet transactions to all peers.\n" - "Intended only for testing; the wallet code periodically re-broadcasts\n" - "automatically.\n", - {}, - RPCResult{ - "Returns an RPC error if -walletbroadcast is set to false.\n" - "Returns array of transaction ids that were re-broadcast.\n" - }, - RPCExamples{""}, - }.ToString() - ); - - if (!g_connman) - throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - - auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - - if (!pwallet->GetBroadcastTransactions()) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast"); - } - - std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(*locked_chain, GetTime(), g_connman.get()); - UniValue result(UniValue::VARR); - for (const uint256& txid : txids) - { - result.push_back(txid.ToString()); - } - return result; -} - static UniValue listunspent(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -2994,7 +2945,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("feeRate")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); } - coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"]); + coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"], pwallet->chain().estimateMaxBlocks()); } if (options.exists("estimate_mode")) { if (options.exists("feeRate")) { @@ -3152,7 +3103,7 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -3199,7 +3150,14 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - return SignTransaction(pwallet->chain(), mtx, request.params[1], pwallet, false, request.params[2]); + // Fetch previous transactions (inputs): + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : mtx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + pwallet->chain().findCoins(coins); + + return SignTransaction(mtx, request.params[1], pwallet, coins, false, request.params[2]); } static UniValue bumpfee(const JSONRPCRequest& request) @@ -3216,9 +3174,9 @@ static UniValue bumpfee(const JSONRPCRequest& request) RPCHelpMan{"bumpfee", "\nBumps the fee of an opt-in-RBF transaction T, replacing it with a new transaction B.\n" "An opt-in RBF transaction with the given txid must be in the wallet.\n" - "The command will pay the additional fee by decreasing (or perhaps removing) its change output.\n" - "If the change output is not big enough to cover the increased fee, the command will currently fail\n" - "instead of adding new inputs to compensate. (A future implementation could improve this.)\n" + "The command will pay the additional fee by reducing change outputs or adding inputs when necessary. It may add a new change output if one does not already exist.\n" + "If `totalFee` is given, adding inputs is not supported, so there must be a single change output that is big enough or it will fail.\n" + "All inputs in the original transaction will be included in the replacement transaction.\n" "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" "By default, the new fee will be calculated automatically using estimatesmartfee.\n" "The user can specify a confirmation target for estimatesmartfee.\n" @@ -3284,7 +3242,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { // TODO: alias this to conf_target - coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"]); + coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"], pwallet->chain().estimateMaxBlocks()); } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); if (totalFee <= 0) { @@ -3315,7 +3273,14 @@ static UniValue bumpfee(const JSONRPCRequest& request) CAmount old_fee; CAmount new_fee; CMutableTransaction mtx; - feebumper::Result res = feebumper::CreateTransaction(pwallet, hash, coin_control, totalFee, errors, old_fee, new_fee, mtx); + feebumper::Result res; + if (totalFee > 0) { + // Targeting total fee bump. Requires a change output of sufficient size. + res = feebumper::CreateTotalBumpTransaction(pwallet, hash, coin_control, totalFee, errors, old_fee, new_fee, mtx); + } else { + // Targeting feerate bump. + res = feebumper::CreateRateBumpTransaction(pwallet, hash, coin_control, errors, old_fee, new_fee, mtx); + } if (res != feebumper::Result::OK) { switch(res) { case feebumper::Result::INVALID_ADDRESS_OR_KEY: @@ -3358,62 +3323,6 @@ static UniValue bumpfee(const JSONRPCRequest& request) return result; } -UniValue generate(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"generate", - "\nMine up to nblocks blocks immediately (before the RPC call returns) to an address in the wallet.\n", - { - {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, - {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, - }, - RPCResult{ - "[ blockhashes ] (array) hashes of blocks generated\n" - }, - RPCExamples{ - "\nGenerate 11 blocks\n" - + HelpExampleCli("generate", "11") - }, - }.ToString()); - } - - if (!IsDeprecatedRPCEnabled("generate")) { - throw JSONRPCError(RPC_METHOD_DEPRECATED, "The wallet generate rpc method is deprecated and will be fully removed in v0.19. " - "To use generate in v0.18, restart bitcoind with -deprecatedrpc=generate.\n" - "Clients should transition to using the node rpc method generatetoaddress\n"); - } - - int num_generate = request.params[0].get_int(); - uint64_t max_tries = 1000000; - if (!request.params[1].isNull()) { - max_tries = request.params[1].get_int(); - } - - std::shared_ptr<CReserveScript> coinbase_script; - pwallet->GetScriptForMining(coinbase_script); - - // If the keypool is exhausted, no script is returned at all. Catch this. - if (!coinbase_script) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); - } - - //throw an error if no script was provided - if (coinbase_script->reserveScript.empty()) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "No coinbase script available"); - } - - return generateBlocks(coinbase_script, num_generate, max_tries, true); -} - UniValue rescanblockchain(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -3513,7 +3422,7 @@ class DescribeWalletAddressVisitor : public boost::static_visitor<UniValue> public: CWallet * const pwallet; - void ProcessSubScript(const CScript& subscript, UniValue& obj, bool include_addresses = false) const + void ProcessSubScript(const CScript& subscript, UniValue& obj) const { // Always present: script type and redeemscript std::vector<std::vector<unsigned char>> solutions_data; @@ -3522,7 +3431,6 @@ public: obj.pushKV("hex", HexStr(subscript.begin(), subscript.end())); CTxDestination embedded; - UniValue a(UniValue::VARR); if (ExtractDestination(subscript, embedded)) { // Only when the script corresponds to an address. UniValue subobj(UniValue::VOBJ); @@ -3535,7 +3443,6 @@ public: // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. if (subobj.exists("pubkey")) obj.pushKV("pubkey", subobj["pubkey"]); obj.pushKV("embedded", std::move(subobj)); - if (include_addresses) a.push_back(EncodeDestination(embedded)); } else if (which_type == TX_MULTISIG) { // Also report some information on multisig scripts (which do not have a corresponding address). // TODO: abstract out the common functionality between this logic and ExtractDestinations. @@ -3543,17 +3450,10 @@ public: UniValue pubkeys(UniValue::VARR); for (size_t i = 1; i < solutions_data.size() - 1; ++i) { CPubKey key(solutions_data[i].begin(), solutions_data[i].end()); - if (include_addresses) a.push_back(EncodeDestination(key.GetID())); pubkeys.push_back(HexStr(key.begin(), key.end())); } obj.pushKV("pubkeys", std::move(pubkeys)); } - - // The "addresses" field is confusing because it refers to public keys using their P2PKH address. - // For that reason, only add the 'addresses' field when needed for backward compatibility. New applications - // can use the 'embedded'->'address' field for P2SH or P2WSH wrapped addresses, and 'pubkeys' for - // inspecting multisig participants. - if (include_addresses) obj.pushKV("addresses", std::move(a)); } explicit DescribeWalletAddressVisitor(CWallet* _pwallet) : pwallet(_pwallet) {} @@ -3576,7 +3476,7 @@ public: UniValue obj(UniValue::VOBJ); CScript subscript; if (pwallet && pwallet->GetCScript(scriptID, subscript)) { - ProcessSubScript(subscript, obj, IsDeprecatedRPCEnabled("validateaddress")); + ProcessSubScript(subscript, obj); } return obj; } @@ -3790,9 +3690,20 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); + std::set<std::string> addresses; for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { if (item.second.name == label) { - ret.pushKV(EncodeDestination(item.first), AddressBookDataToJSON(item.second, false)); + std::string address = EncodeDestination(item.first); + // CWallet::mapAddressBook is not expected to contain duplicate + // address strings, but build a separate set as a precaution just in + // case it does. + bool unique = addresses.emplace(address).second; + assert(unique); + // UniValue::pushKV checks if the key exists in O(N) + // and since duplicate addresses are unexpected (checked with + // std::set in O(log(N))), UniValue::__pushKV is used instead, + // which currently is O(1). + ret.__pushKV(address, AddressBookDataToJSON(item.second, false)); } } @@ -3894,7 +3805,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }.ToString()); } - if (IsInitialBlockDownload()) { + if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); } @@ -3939,19 +3850,6 @@ UniValue sethdseed(const JSONRPCRequest& request) return NullUniValue; } -void AddKeypathToMap(const CWallet* pwallet, const CKeyID& keyID, std::map<CPubKey, KeyOriginInfo>& hd_keypaths) -{ - CPubKey vchPubKey; - if (!pwallet->GetPubKey(keyID, vchPubKey)) { - return; - } - KeyOriginInfo info; - if (!pwallet->GetKeyOrigin(keyID, info)) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal keypath is broken"); - } - hd_keypaths.emplace(vchPubKey, std::move(info)); -} - UniValue walletprocesspsbt(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -4156,8 +4054,6 @@ UniValue importmulti(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "generating", "generate", &generate, {"nblocks","maxtries"} }, - { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, @@ -4215,8 +4111,8 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterWalletRPCCommands(CRPCTable &t) +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) { for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + handlers.emplace_back(chain.handleRpc(commands[vcidx])); } diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 58053bde59..7cf607ccc7 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_WALLET_RPCWALLET_H #define BITCOIN_WALLET_RPCWALLET_H +#include <memory> #include <string> +#include <vector> class CRPCTable; class CWallet; @@ -14,7 +16,12 @@ class UniValue; struct PartiallySignedTransaction; class CTransaction; -void RegisterWalletRPCCommands(CRPCTable &t); +namespace interfaces { +class Chain; +class Handler; +} + +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers); /** * Figures out what wallet, if any, to use for a JSONRPCRequest. diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 5c65acf601..34b9770e8b 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -8,7 +8,7 @@ #include <amount.h> #include <primitives/transaction.h> #include <random.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> @@ -29,7 +29,7 @@ typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; static auto testChain = interfaces::MakeChain(); -static CWallet testWallet(*testChain, WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); @@ -69,8 +69,7 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); if (fIsFromMe) { - wtx->fDebitCached = true; - wtx->nDebitCached = 1; + wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); } COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); @@ -115,7 +114,7 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->fDebitCached && coin.tx->nDebitCached == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); + for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); return static_groups; } @@ -135,7 +134,6 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) ///////////////////////// // Known Outcome tests // ///////////////////////// - BOOST_TEST_MESSAGE("Testing known outcomes"); // Empty utxo pool BOOST_CHECK(!SelectCoinsBnB(GroupCoins(utxo_pool), 1 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 2a64749379..d9b07af329 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -7,7 +7,7 @@ #include <boost/test/unit_test.hpp> #include <fs.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <wallet/db.h> diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index cd47b31da1..0f2d9fbd3d 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -6,7 +6,7 @@ #define BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H #include <interfaces/chain.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> struct InitWalletDirTestingSetup: public BasicTestingSetup { diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 5852d3ef84..e1c53c83e2 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -4,7 +4,7 @@ #include <boost/test/unit_test.hpp> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <wallet/test/init_test_fixture.h> #include <init.h> diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 789e86e21b..f774cb4ad1 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -12,7 +12,7 @@ #include <univalue.h> #include <boost/test/unit_test.hpp> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <wallet/test/wallet_test_fixture.h> BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index ae7092fa89..acc61c984f 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <util/strencodings.h> #include <wallet/crypter.h> diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a5fb1db86c..6526e69eea 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -8,17 +8,13 @@ #include <wallet/db.h> #include <wallet/rpcwallet.h> -WalletTestingSetup::WalletTestingSetup(const std::string& chainName): - TestingSetup(chainName), m_wallet(*m_chain, WalletLocation(), WalletDatabase::CreateMock()) +WalletTestingSetup::WalletTestingSetup(const std::string& chainName) + : TestingSetup(chainName), + m_wallet(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()) { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); - RegisterValidationInterface(&m_wallet); + m_wallet.handleNotifications(); - RegisterWalletRPCCommands(tableRPC); -} - -WalletTestingSetup::~WalletTestingSetup() -{ - UnregisterValidationInterface(&m_wallet); + m_chain_client->registerRpcs(); } diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index e6fe8c9473..1017e61700 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H #define BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <interfaces/chain.h> #include <interfaces/wallet.h> @@ -17,9 +17,9 @@ */ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - ~WalletTestingSetup(); std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); CWallet m_wallet; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index e674b2faea..3cdbde33c3 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -13,7 +13,7 @@ #include <consensus/validation.h> #include <interfaces/chain.h> #include <rpc/server.h> -#include <test/test_bitcoin.h> +#include <test/setup_common.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/test/wallet_test_fixture.h> @@ -49,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions accommodates a null start block. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -58,13 +58,13 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 0); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); } // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -73,7 +73,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 100 * COIN); } // Prune the older block file. @@ -83,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -92,7 +92,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 50 * COIN); } // Prune the remaining block file. @@ -101,7 +101,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -110,7 +110,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 0); + BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); } } @@ -135,7 +135,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -198,7 +198,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(wallet->cs_wallet); wallet->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); @@ -214,7 +214,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); JSONRPCRequest request; request.params.setArray(); @@ -245,7 +245,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { auto chain = interfaces::MakeChain(); - CWallet wallet(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); LOCK(wallet.cs_wallet); @@ -340,7 +340,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique<CWallet>(*m_chain, WalletLocation(), WalletDatabase::CreateMock()); + wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); @@ -368,7 +368,7 @@ public: CCoinControl dummy; BOOST_CHECK(wallet->CreateTransaction(*m_locked_chain, {recipient}, tx, reservekey, fee, changePos, error, dummy)); CValidationState state; - BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, nullptr, state)); + BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, state)); CMutableTransaction blocktx; { LOCK(wallet->cs_wallet); @@ -451,7 +451,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 388422bec8..071ae23e1b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1,21 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <wallet/wallet.h> -#include <checkpoints.h> #include <chain.h> -#include <wallet/coincontrol.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <fs.h> #include <interfaces/chain.h> +#include <interfaces/wallet.h> #include <key.h> #include <key_io.h> #include <keystore.h> -#include <validation.h> #include <net.h> #include <policy/fees.h> #include <policy/policy.h> @@ -28,7 +26,13 @@ #include <timedata.h> #include <txmempool.h> #include <util/bip32.h> +#include <util/error.h> +#include <util/fees.h> #include <util/moneystr.h> +#include <util/rbf.h> +#include <util/validation.h> +#include <validation.h> +#include <wallet/coincontrol.h> #include <wallet/fees.h> #include <algorithm> @@ -95,7 +99,7 @@ static void ReleaseWallet(CWallet* wallet) wallet->WalletLogPrintf("Releasing wallet\n"); wallet->BlockUntilSyncedToCurrentChain(); wallet->Flush(); - UnregisterValidationInterface(wallet); + wallet->m_chain_notifications_handler.reset(); delete wallet; // Wallet is now released, notify UnloadWallet, if any. { @@ -192,7 +196,7 @@ CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; @@ -280,9 +284,9 @@ void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } -bool CWallet::AddKeyPubKeyWithDB(WalletBatch &batch, const CKey& secret, const CPubKey &pubkey) +bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); // Make sure we aren't adding private keys to private key disabled wallets assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); @@ -344,16 +348,16 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, } } -void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &meta) +void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; } -void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &meta) +void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // m_script_metadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); m_script_metadata[script_id] = meta; } @@ -366,7 +370,7 @@ bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, void CWallet::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } @@ -568,7 +572,7 @@ void CWallet::ChainStateFlushed(const CBlockLocator& loc) void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool fExplicit) { - LOCK(cs_wallet); // nWalletVersion + LOCK(cs_wallet); if (nWalletVersion >= nVersion) return; @@ -592,7 +596,7 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool CWallet::SetMaxVersion(int nVersion) { - LOCK(cs_wallet); // nWalletVersion, nWalletMaxVersion + LOCK(cs_wallet); // cannot downgrade below current version if (nWalletVersion > nVersion) return false; @@ -876,9 +880,9 @@ DBErrors CWallet::ReorderTransactions() return DBErrors::LOAD_OK; } -int64_t CWallet::IncOrderPosNext(WalletBatch *batch) +int64_t CWallet::IncOrderPosNext(WalletBatch* batch) { - AssertLockHeld(cs_wallet); // nOrderPosNext + AssertLockHeld(cs_wallet); int64_t nRet = nOrderPosNext++; if (batch) { batch->WriteOrderPosNext(nOrderPosNext); @@ -940,7 +944,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.BindWallet(this); bool fInsertedNew = ret.second; if (fInsertedNew) { - wtx.nTimeReceived = GetAdjustedTime(); + wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); wtx.nTimeSmart = ComputeTimeSmart(wtx); @@ -1242,7 +1246,8 @@ void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { } } -void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { +void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) { + const uint256& block_hash = block.GetHash(); auto locked_chain = chain().lock(); LOCK(cs_wallet); // TODO: Temporarily ensure that mempool removals are notified before @@ -1257,46 +1262,37 @@ void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const SyncTransaction(ptx, {} /* block hash */, 0 /* position in block */); TransactionRemovedFromMempool(ptx); } - for (size_t i = 0; i < pblock->vtx.size(); i++) { - SyncTransaction(pblock->vtx[i], pindex->GetBlockHash(), i); - TransactionRemovedFromMempool(pblock->vtx[i]); + for (size_t i = 0; i < block.vtx.size(); i++) { + SyncTransaction(block.vtx[i], block_hash, i); + TransactionRemovedFromMempool(block.vtx[i]); } - m_last_block_processed = pindex->GetBlockHash(); + m_last_block_processed = block_hash; } -void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { +void CWallet::BlockDisconnected(const CBlock& block) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (const CTransactionRef& ptx : pblock->vtx) { + for (const CTransactionRef& ptx : block.vtx) { SyncTransaction(ptx, {} /* block hash */, 0 /* position in block */); } } +void CWallet::UpdatedBlockTip() +{ + m_best_block_time = GetTime(); +} void CWallet::BlockUntilSyncedToCurrentChain() { - AssertLockNotHeld(cs_main); AssertLockNotHeld(cs_wallet); - - { - // Skip the queue-draining stuff if we know we're caught up with - // chainActive.Tip()... - // We could also take cs_wallet here, and call m_last_block_processed - // protected by cs_wallet instead of cs_main, but as long as we need - // cs_main here anyway, it's easier to just call it cs_main-protected. - auto locked_chain = chain().lock(); - - if (!m_last_block_processed.IsNull() && locked_chain->isPotentialTip(m_last_block_processed)) { - return; - } - } - - // ...otherwise put a callback in the validation interface queue and wait + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip(), otherwise put a callback in the validation interface queue and wait // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). - SyncWithValidationInterfaceQueue(); + uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); + chain().waitForNotificationsIfNewBlocksConnected(last_block_hash); } @@ -1720,7 +1716,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r uint256 start_block; { auto locked_chain = chain().lock(); - const Optional<int> start_height = locked_chain->findFirstBlockWithTime(startTime - TIMESTAMP_WINDOW, &start_block); + const Optional<int> start_height = locked_chain->findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, &start_block); const Optional<int> tip_height = locked_chain->getHeight(); WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, tip_height && start_height ? *tip_height - *start_height + 1 : 0); } @@ -1763,6 +1759,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, const uint256& stop_block, const WalletRescanReserver& reserver, bool fUpdate) { int64_t nNow = GetTime(); + int64_t start_time = GetTimeMillis(); assert(reserver.isReserved()); @@ -1771,101 +1768,99 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); + fAbortRescan = false; + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup + uint256 tip_hash; + // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. + Optional<int> block_height = MakeOptional(false, int()); + double progress_begin; + double progress_end; { - fAbortRescan = false; - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - uint256 tip_hash; - // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> block_height = MakeOptional(false, int()); - double progress_begin; - double progress_end; - { - auto locked_chain = chain().lock(); - if (Optional<int> tip_height = locked_chain->getHeight()) { - tip_hash = locked_chain->getBlockHash(*tip_height); - } - block_height = locked_chain->getBlockHeight(block_hash); - progress_begin = chain().guessVerificationProgress(block_hash); - progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); - } - double progress_current = progress_begin; - while (block_height && !fAbortRescan && !ShutdownRequested()) { - if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), std::max(1, std::min(99, (int)((progress_current - progress_begin) / (progress_end - progress_begin) * 100)))); - } - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); - } + auto locked_chain = chain().lock(); + if (Optional<int> tip_height = locked_chain->getHeight()) { + tip_hash = locked_chain->getBlockHash(*tip_height); + } + block_height = locked_chain->getBlockHeight(block_hash); + progress_begin = chain().guessVerificationProgress(block_hash); + progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); + } + double progress_current = progress_begin; + while (block_height && !fAbortRescan && !chain().shutdownRequested()) { + if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), std::max(1, std::min(99, (int)((progress_current - progress_begin) / (progress_end - progress_begin) * 100)))); + } + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); + } - CBlock block; - if (chain().findBlock(block_hash, &block) && !block.IsNull()) { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - if (!locked_chain->getBlockHeight(block_hash)) { - // Abort scan if current block is no longer active, to prevent - // marking transactions as coming from the wrong block. - // TODO: This should return success instead of failure, see - // https://github.com/bitcoin/bitcoin/pull/14711#issuecomment-458342518 - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; - break; - } - for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], block_hash, posInBlock, fUpdate); - } - // scan succeeded, record block as most recent successfully scanned - result.last_scanned_block = block_hash; - result.last_scanned_height = *block_height; - } else { - // could not scan block, keep scanning but record this block as the most recent failure + CBlock block; + if (chain().findBlock(block_hash, &block) && !block.IsNull()) { + auto locked_chain = chain().lock(); + LOCK(cs_wallet); + if (!locked_chain->getBlockHeight(block_hash)) { + // Abort scan if current block is no longer active, to prevent + // marking transactions as coming from the wrong block. + // TODO: This should return success instead of failure, see + // https://github.com/bitcoin/bitcoin/pull/14711#issuecomment-458342518 result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; + break; } - if (block_hash == stop_block) { + for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { + SyncTransaction(block.vtx[posInBlock], block_hash, posInBlock, fUpdate); + } + // scan succeeded, record block as most recent successfully scanned + result.last_scanned_block = block_hash; + result.last_scanned_height = *block_height; + } else { + // could not scan block, keep scanning but record this block as the most recent failure + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; + } + if (block_hash == stop_block) { + break; + } + { + auto locked_chain = chain().lock(); + Optional<int> tip_height = locked_chain->getHeight(); + if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { + // break successfully when rescan has reached the tip, or + // previous block is no longer on the chain due to a reorg break; } - { - auto locked_chain = chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); - if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { - // break successfully when rescan has reached the tip, or - // previous block is no longer on the chain due to a reorg - break; - } - // increment block and verification progress - block_hash = locked_chain->getBlockHash(++*block_height); - progress_current = chain().guessVerificationProgress(block_hash); + // increment block and verification progress + block_hash = locked_chain->getBlockHash(++*block_height); + progress_current = chain().guessVerificationProgress(block_hash); - // handle updated tip hash - const uint256 prev_tip_hash = tip_hash; - tip_hash = locked_chain->getBlockHash(*tip_height); - if (stop_block.IsNull() && prev_tip_hash != tip_hash) { - // in case the tip has changed, update progress max - progress_end = chain().guessVerificationProgress(tip_hash); - } + // handle updated tip hash + const uint256 prev_tip_hash = tip_hash; + tip_hash = locked_chain->getBlockHash(*tip_height); + if (stop_block.IsNull() && prev_tip_hash != tip_hash) { + // in case the tip has changed, update progress max + progress_end = chain().guessVerificationProgress(tip_hash); } } - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 100); // hide progress dialog in GUI - if (block_height && fAbortRescan) { - WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); - result.status = ScanResult::USER_ABORT; - } else if (block_height && ShutdownRequested()) { - WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); - result.status = ScanResult::USER_ABORT; - } + } + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 100); // hide progress dialog in GUI + if (block_height && fAbortRescan) { + WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); + result.status = ScanResult::USER_ABORT; + } else if (block_height && chain().shutdownRequested()) { + WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); + result.status = ScanResult::USER_ABORT; + } else { + WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); } return result; } -void CWallet::ReacceptWalletTransactions() +void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) return; - auto locked_chain = chain().lock(); - LOCK(cs_wallet); std::map<int64_t, CWalletTx*> mapSorted; // Sort pending wallet transactions based on their initial wallet insertion order @@ -1875,7 +1870,7 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(*locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1886,30 +1881,31 @@ void CWallet::ReacceptWalletTransactions() for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); CValidationState state; - wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state); + wtx.AcceptToMemoryPool(locked_chain, state); } } -bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman) +bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain) { - assert(pwallet->GetBroadcastTransactions()); - if (!IsCoinBase() && !isAbandoned() && GetDepthInMainChain(locked_chain) == 0) - { - CValidationState state; - /* GetDepthInMainChain already catches known conflicts. */ - if (InMempool() || AcceptToMemoryPool(locked_chain, maxTxFee, state)) { - pwallet->WalletLogPrintf("Relaying wtx %s\n", GetHash().ToString()); - if (connman) { - CInv inv(MSG_TX, GetHash()); - connman->ForEachNode([&inv](CNode* pnode) - { - pnode->PushInventory(inv); - }); - return true; - } - } - } - return false; + // Can't relay if wallet is not broadcasting + if (!pwallet->GetBroadcastTransactions()) return false; + // Don't relay coinbase transactions outside blocks + if (IsCoinBase()) return false; + // Don't relay abandoned transactions + if (isAbandoned()) return false; + // Don't relay conflicted or already confirmed transactions + if (GetDepthInMainChain(locked_chain) != 0) return false; + // Don't relay transactions that aren't accepted to the mempool + CValidationState unused_state; + if (!InMempool() && !AcceptToMemoryPool(locked_chain, unused_state)) return false; + // Don't try to relay if the node is not connected to the p2p network + if (!pwallet->chain().p2pEnabled()) return false; + + // Try to relay the transaction + pwallet->WalletLogPrintf("Relaying wtx %s\n", GetHash().ToString()); + pwallet->chain().relayTransaction(GetHash()); + + return true; } std::set<uint256> CWalletTx::GetConflicts() const @@ -1924,33 +1920,26 @@ std::set<uint256> CWalletTx::GetConflicts() const return result; } +CAmount CWalletTx::GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate) const +{ + auto& amount = m_amounts[type]; + if (recalculate || !amount.m_cached[filter]) { + amount.Set(filter, type == DEBIT ? pwallet->GetDebit(*tx, filter) : pwallet->GetCredit(*tx, filter)); + } + return amount.m_value[filter]; +} + CAmount CWalletTx::GetDebit(const isminefilter& filter) const { if (tx->vin.empty()) return 0; CAmount debit = 0; - if(filter & ISMINE_SPENDABLE) - { - if (fDebitCached) - debit += nDebitCached; - else - { - nDebitCached = pwallet->GetDebit(*tx, ISMINE_SPENDABLE); - fDebitCached = true; - debit += nDebitCached; - } + if (filter & ISMINE_SPENDABLE) { + debit += GetCachableAmount(DEBIT, ISMINE_SPENDABLE); } - if(filter & ISMINE_WATCH_ONLY) - { - if(fWatchDebitCached) - debit += nWatchDebitCached; - else - { - nWatchDebitCached = pwallet->GetDebit(*tx, ISMINE_WATCH_ONLY); - fWatchDebitCached = true; - debit += nWatchDebitCached; - } + if (filter & ISMINE_WATCH_ONLY) { + debit += GetCachableAmount(DEBIT, ISMINE_WATCH_ONLY); } return debit; } @@ -1962,28 +1951,12 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) - { + if (filter & ISMINE_SPENDABLE) { // GetBalance can assume transactions in mapWallet won't change - if (fCreditCached) - credit += nCreditCached; - else - { - nCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); - fCreditCached = true; - credit += nCreditCached; - } + credit += GetCachableAmount(CREDIT, ISMINE_SPENDABLE); } - if (filter & ISMINE_WATCH_ONLY) - { - if (fWatchCreditCached) - credit += nWatchCreditCached; - else - { - nWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); - fWatchCreditCached = true; - credit += nWatchCreditCached; - } + if (filter & ISMINE_WATCH_ONLY) { + credit += GetCachableAmount(CREDIT, ISMINE_WATCH_ONLY); } return credit; } @@ -1991,11 +1964,7 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine CAmount CWalletTx::GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache) const { if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { - if (fUseCache && fImmatureCreditCached) - return nImmatureCreditCached; - nImmatureCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); - fImmatureCreditCached = true; - return nImmatureCreditCached; + return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; @@ -2006,23 +1975,15 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo if (pwallet == nullptr) return 0; + // Avoid caching ismine for NO or ALL cases (could remove this check and simplify in the future). + bool allow_cache = filter == ISMINE_SPENDABLE || filter == ISMINE_WATCH_ONLY; + // Must wait until coinbase is safely deep enough in the chain before valuing it if (IsImmatureCoinBase(locked_chain)) return 0; - CAmount* cache = nullptr; - bool* cache_used = nullptr; - - if (filter == ISMINE_SPENDABLE) { - cache = &nAvailableCreditCached; - cache_used = &fAvailableCreditCached; - } else if (filter == ISMINE_WATCH_ONLY) { - cache = &nAvailableWatchCreditCached; - cache_used = &fAvailableWatchCreditCached; - } - - if (fUseCache && cache_used && *cache_used) { - return *cache; + if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { + return m_amounts[AVAILABLE_CREDIT].m_value[filter]; } CAmount nCredit = 0; @@ -2038,22 +1999,17 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo } } - if (cache) { - *cache = nCredit; - assert(cache_used); - *cache_used = true; + if (allow_cache) { + m_amounts[AVAILABLE_CREDIT].Set(filter, nCredit); } + return nCredit; } CAmount CWalletTx::GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache) const { if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { - if (fUseCache && fImmatureWatchCreditCached) - return nImmatureWatchCreditCached; - nImmatureWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); - fImmatureWatchCreditCached = true; - return nImmatureWatchCreditCached; + return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } return 0; @@ -2075,11 +2031,10 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Quick answer in most cases - if (!CheckFinalTx(*tx)) + if (!locked_chain.checkFinalTx(*tx)) { return false; + } int nDepth = GetDepthInMainChain(locked_chain); if (nDepth >= 1) return true; @@ -2115,185 +2070,95 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const return CTransaction(tx1) == CTransaction(tx2); } -std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman) +// Rebroadcast transactions from the wallet. We do this on a random timer +// to slightly obfuscate which transactions come from our wallet. +// +// Ideally, we'd only resend transactions that we think should have been +// mined in the most recent block. Any transaction that wasn't in the top +// blockweight of transactions in the mempool shouldn't have been mined, +// and so is probably just sitting in the mempool waiting to be confirmed. +// Rebroadcasting does nothing to speed up confirmation and only damages +// privacy. +void CWallet::ResendWalletTransactions() { - std::vector<uint256> result; - - LOCK(cs_wallet); - - // Sort them in chronological order - std::multimap<unsigned int, CWalletTx*> mapSorted; - for (std::pair<const uint256, CWalletTx>& item : mapWallet) - { - CWalletTx& wtx = item.second; - // Don't rebroadcast if newer than nTime: - if (wtx.nTimeReceived > nTime) - continue; - mapSorted.insert(std::make_pair(wtx.nTimeReceived, &wtx)); - } - for (const std::pair<const unsigned int, CWalletTx*>& item : mapSorted) - { - CWalletTx& wtx = *item.second; - if (wtx.RelayWalletTransaction(locked_chain, connman)) - result.push_back(wtx.GetHash()); - } - return result; -} + // During reindex, importing and IBD, old wallet transactions become + // unconfirmed. Don't resend them as that would spam other nodes. + if (!chain().isReadyToBroadcast()) return; -void CWallet::ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) -{ // Do this infrequently and randomly to avoid giving away // that these are our transactions. - if (GetTime() < nNextResend || !fBroadcastTransactions) - return; + if (GetTime() < nNextResend || !fBroadcastTransactions) return; bool fFirst = (nNextResend == 0); nNextResend = GetTime() + GetRand(30 * 60); - if (fFirst) - return; + if (fFirst) return; // Only do it if there's been a new block since last time - if (nBestBlockTime < nLastResend) - return; + if (m_best_block_time < nLastResend) return; nLastResend = GetTime(); - // Rebroadcast unconfirmed txes older than 5 minutes before the last - // block was found: - auto locked_chain = chain().assumeLocked(); // Temporary. Removed in upcoming lock cleanup - std::vector<uint256> relayed = ResendWalletTransactionsBefore(*locked_chain, nBestBlockTime-5*60, connman); - if (!relayed.empty()) - WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed.size()); -} - -/** @} */ // end of mapWallet - - - - -/** @defgroup Actions - * - * @{ - */ - + int relayed_tx_count = 0; -CAmount CWallet::GetBalance(const isminefilter& filter, const int min_depth) const -{ - CAmount nTotal = 0; - { + { // locked_chain and cs_wallet scope auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) >= min_depth) { - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, filter); - } - } - } - return nTotal; -} - -CAmount CWallet::GetUnconfirmedBalance() const -{ - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain); + // Relay transactions + for (std::pair<const uint256, CWalletTx>& item : mapWallet) { + CWalletTx& wtx = item.second; + // only rebroadcast unconfirmed txes older than 5 minutes before the + // last block was found + if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; + if (wtx.RelayWalletTransaction(*locked_chain)) ++relayed_tx_count; } - } - return nTotal; -} + } // locked_chain and cs_wallet -CAmount CWallet::GetImmatureBalance() const -{ - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureCredit(*locked_chain); - } + if (relayed_tx_count > 0) { + WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed_tx_count); } - return nTotal; } -CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const +/** @} */ // end of mapWallet + +void MaybeResendWalletTxs() { - CAmount nTotal = 0; - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - for (const auto& entry : mapWallet) - { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, ISMINE_WATCH_ONLY); - } + for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + pwallet->ResendWalletTransactions(); } - return nTotal; } -CAmount CWallet::GetImmatureWatchOnlyBalance() const + +/** @defgroup Actions + * + * @{ + */ + + +CWallet::Balance CWallet::GetBalance(const int min_depth) const { - CAmount nTotal = 0; + Balance ret; { auto locked_chain = chain().lock(); LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureWatchOnlyCredit(*locked_chain); - } - } - return nTotal; -} - -// Calculate total balance in a different way from GetBalance. The biggest -// difference is that GetBalance sums up all unspent TxOuts paying to the -// wallet, while this sums up both spent and unspent TxOuts paying to the -// wallet, and then subtracts the values of TxIns spending from the wallet. This -// also has fewer restrictions on which unconfirmed transactions are considered -// trusted. -CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth) const -{ - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - - CAmount balance = 0; - for (const auto& entry : mapWallet) { - const CWalletTx& wtx = entry.second; - const int depth = wtx.GetDepthInMainChain(*locked_chain); - if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.IsImmatureCoinBase(*locked_chain)) { - continue; - } - - // Loop through tx outputs and add incoming payments. For outgoing txs, - // treat change outputs specially, as part of the amount debited. - CAmount debit = wtx.GetDebit(filter); - const bool outgoing = debit > 0; - for (const CTxOut& out : wtx.tx->vout) { - if (outgoing && IsChange(out)) { - debit -= out.nValue; - } else if (IsMine(out) & filter && depth >= minDepth) { - balance += out.nValue; + const CWalletTx& wtx = entry.second; + const bool is_trusted{wtx.IsTrusted(*locked_chain)}; + const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; + const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE)}; + const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY)}; + if (is_trusted && tx_depth >= min_depth) { + ret.m_mine_trusted += tx_credit_mine; + ret.m_watchonly_trusted += tx_credit_watchonly; } - } - - // For outgoing txs, subtract amount debited. - if (outgoing) { - balance -= debit; + if (!is_trusted && tx_depth == 0 && wtx.InMempool()) { + ret.m_mine_untrusted_pending += tx_credit_mine; + ret.m_watchonly_untrusted_pending += tx_credit_watchonly; + } + ret.m_mine_immature += wtx.GetImmatureCredit(*locked_chain); + ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(*locked_chain); } } - - return balance; + return ret; } CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const @@ -2314,7 +2179,6 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t nMaximumCount, const int nMinDepth, const int nMaxDepth) const { - AssertLockHeld(cs_main); AssertLockHeld(cs_wallet); vCoins.clear(); @@ -2323,24 +2187,25 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; - const CWalletTx* pcoin = &entry.second; + const CWalletTx& wtx = entry.second; - if (!CheckFinalTx(*pcoin->tx)) + if (!locked_chain.checkFinalTx(*wtx.tx)) { continue; + } - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < 0) continue; // We should not consider coins which aren't at least in our mempool // It's possible for these to be conflicted via ancestors which we may never be able to detect - if (nDepth == 0 && !pcoin->InMempool()) + if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = pcoin->IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain); // We should not consider coins from transactions that are replacing // other transactions. @@ -2357,7 +2222,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // be a 1-block reorg away from the chain where transactions A and C // were accepted to another chain where B, B', and C were all // accepted. - if (nDepth == 0 && pcoin->mapValue.count("replaces_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaces_txid")) { safeTx = false; } @@ -2369,7 +2234,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // intending to replace A', but potentially resulting in a scenario // where A, A', and D could all be accepted (instead of just B and // D, or just A and A' like the user would want). - if (nDepth == 0 && pcoin->mapValue.count("replaced_by_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaced_by_txid")) { safeTx = false; } @@ -2380,8 +2245,8 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth < nMinDepth || nDepth > nMaxDepth) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { - if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + if (wtx.tx->vout[i].nValue < nMinimumAmount || wtx.tx->vout[i].nValue > nMaximumAmount) continue; if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) @@ -2393,20 +2258,20 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsSpent(locked_chain, wtxid, i)) continue; - isminetype mine = IsMine(pcoin->tx->vout[i]); + isminetype mine = IsMine(wtx.tx->vout[i]); if (mine == ISMINE_NO) { continue; } - bool solvable = IsSolvable(*this, pcoin->tx->vout[i].scriptPubKey); + bool solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey); bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); - vCoins.push_back(COutput(pcoin, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { - nTotal += pcoin->tx->vout[i].nValue; + nTotal += wtx.tx->vout[i].nValue; if (nTotal >= nMinimumSumAmount) { return; @@ -2423,7 +2288,6 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Chain::Lock& locked_chain) const { - AssertLockHeld(cs_main); AssertLockHeld(cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; @@ -2488,10 +2352,10 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil FeeCalculation feeCalc; CCoinControl temp; temp.m_confirm_target = 1008; - CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this, ::feeEstimator).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); // Filter by the min conf specs and add to utxo_pool and calculate effective value for (OutputGroup& group : groups) { @@ -2565,13 +2429,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { - const CWalletTx* pcoin = &it->second; + const CWalletTx& wtx = it->second; // Clearly invalid input, fail - if (pcoin->tx->vout.size() <= outpoint.n) + if (wtx.tx->vout.size() <= outpoint.n) return false; // Just to calculate the marginal byte size - nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(CInputCoin(pcoin->tx, outpoint.n)); + nValueFromPresetInputs += wtx.tx->vout[outpoint.n].nValue; + setPresetCoins.insert(CInputCoin(wtx.tx, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -2617,9 +2481,9 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm return res; } -bool CWallet::SignTransaction(CMutableTransaction &tx) +bool CWallet::SignTransaction(CMutableTransaction& tx) { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); // sign the new tx int nIn = 0; @@ -2695,13 +2559,13 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC return true; } -static bool IsCurrentForAntiFeeSniping(interfaces::Chain::Lock& locked_chain) +static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) { - if (IsInitialBlockDownload()) { + if (chain.isInitialBlockDownload()) { return false; } constexpr int64_t MAX_ANTI_FEE_SNIPING_TIP_AGE = 8 * 60 * 60; // in seconds - if (chainActive.Tip()->GetBlockTime() < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { + if (locked_chain.getBlockTime(*locked_chain.getHeight()) < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { return false; } return true; @@ -2711,7 +2575,7 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain::Lock& locked_chain) * Return a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain::Lock& locked_chain) +static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) { uint32_t const height = locked_chain.getHeight().get_value_or(-1); uint32_t locktime; @@ -2735,7 +2599,7 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain::Lock& locked_cha // enough, that fee sniping isn't a problem yet, but by implementing a fix // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. - if (IsCurrentForAntiFeeSniping(locked_chain)) { + if (IsCurrentForAntiFeeSniping(chain, locked_chain)) { locktime = height; // Secondly occasionally randomly pick a nLockTime even further back, so @@ -2809,7 +2673,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std CMutableTransaction txNew; - txNew.nLockTime = GetLocktimeForNewTransaction(locked_chain); + txNew.nLockTime = GetLocktimeForNewTransaction(chain(), locked_chain); FeeCalculation feeCalc; CAmount nFeeNeeded; @@ -2820,7 +2684,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std LOCK(cs_wallet); { std::vector<COutput> vAvailableCoins; - AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control); + AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0, coin_control.m_min_depth); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy // Create change script that will be used if we need change @@ -2861,10 +2725,10 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); - CFeeRate discard_rate = GetDiscardRate(*this, ::feeEstimator); + CFeeRate discard_rate = GetDiscardRate(*this); // Get the fee rate to use effective values in coin selection - CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, &feeCalc); nFeeRet = 0; bool pick_new_inputs = true; @@ -2905,7 +2769,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // Include the fee cost for outputs. Note this is only used for BnB right now coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); - if (IsDust(txout, ::dustRelayFee)) + if (IsDust(txout, chain().relayDustFee())) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2997,7 +2861,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std return false; } - nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, ::mempool, ::feeEstimator, &feeCalc); + nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, &feeCalc); if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { // eventually allow a fallback fee strFailReason = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); @@ -3006,7 +2870,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // If we made it here and we aren't even able to meet the relay fee on the next pass, give up // because we must be at the maximum allowed fee. - if (nFeeNeeded < ::minRelayTxFee.GetFee(nBytes)) + if (nFeeNeeded < chain().relayMinFee().GetFee(nBytes)) { strFailReason = _("Transaction too large for fee policy"); return false; @@ -3025,7 +2889,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // change output. Only try this once. if (nChangePosInOut == -1 && nSubtractFeeFromAmount == 0 && pick_new_inputs) { unsigned int tx_size_with_change = nBytes + coin_selection_params.change_output_size + 2; // Add 2 as a buffer in case increasing # of outputs changes compact size - CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, ::mempool, ::feeEstimator, nullptr); + CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, nullptr); CAmount minimum_value_for_change = GetDustThreshold(change_prototype_txout, discard_rate); if (nFeeRet >= fee_needed_with_change + minimum_value_for_change) { pick_new_inputs = false; @@ -3130,16 +2994,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits - LockPoints lp; - CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); - CTxMemPool::setEntries setAncestors; - size_t nLimitAncestors = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - size_t nLimitAncestorSize = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; - size_t nLimitDescendants = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; - std::string errString; - LOCK(::mempool.cs); - if (!::mempool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { + if (!chain().checkChainLimits(tx)) { strFailReason = _("Transaction has too long of a mempool chain"); return false; } @@ -3159,7 +3014,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state) +bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state) { { auto locked_chain = chain().lock(); @@ -3196,11 +3051,11 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve if (fBroadcastTransactions) { // Broadcast - if (!wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(*locked_chain, state)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", FormatStateMessage(state)); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtx.RelayWalletTransaction(*locked_chain, connman); + wtx.RelayWalletTransaction(*locked_chain); } } } @@ -3209,7 +3064,6 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); fFirstRunRet = false; @@ -3242,8 +3096,8 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - AssertLockHeld(cs_wallet); // mapWallet - DBErrors nZapSelectTxRet = WalletBatch(*database,"cr+").ZapSelectTx(vHashIn, vHashOut); + AssertLockHeld(cs_wallet); + DBErrors nZapSelectTxRet = WalletBatch(*database, "cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); @@ -3269,7 +3123,6 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 MarkDirty(); return DBErrors::LOAD_OK; - } DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) @@ -3389,7 +3242,7 @@ bool CWallet::NewKeyPool() size_t CWallet::KeypoolCountExternalKeys() { - AssertLockHeld(cs_wallet); // setExternalKeyPool + AssertLockHeld(cs_wallet); return setExternalKeyPool.size() + set_pre_split_keypool.size(); } @@ -3619,27 +3472,27 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: LOCK(cs_wallet); for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (!pcoin->IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain)) continue; - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); - if (nDepth < (pcoin->IsFromMe(ISMINE_ALL) ? 0 : 1)) + int nDepth = wtx.GetDepthInMainChain(locked_chain); + if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(pcoin->tx->vout[i])) + if (!IsMine(wtx.tx->vout[i])) continue; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) + if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : pcoin->tx->vout[i].nValue; + CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3653,19 +3506,19 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (pcoin->tx->vin.size() > 0) + if (wtx.tx->vin.size() > 0) { bool any_mine = false; // group all input addresses with each other - for (const CTxIn& txin : pcoin->tx->vin) + for (const CTxIn& txin : wtx.tx->vin) { CTxDestination address; if(!IsMine(txin)) /* If this input isn't mine, ignore it */ @@ -3679,7 +3532,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() // group change with input addresses if (any_mine) { - for (const CTxOut& txout : pcoin->tx->vout) + for (const CTxOut& txout : wtx.tx->vout) if (IsChange(txout)) { CTxDestination txoutAddr; @@ -3696,7 +3549,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (const auto& txout : pcoin->tx->vout) + for (const auto& txout : wtx.tx->vout) if (IsMine(txout)) { CTxDestination address; @@ -3819,38 +3672,27 @@ void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) } } -void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) -{ - std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); - CPubKey pubkey; - if (!rKey->GetReservedKey(pubkey)) - return; - - script = rKey; - script->reserveScript = CScript() << ToByteVector(pubkey) << OP_CHECKSIG; -} - void CWallet::LockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.insert(output); } void CWallet::UnlockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.erase(output); } void CWallet::UnlockAllCoins() { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.clear(); } bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); COutPoint outpt(hash, n); return (setLockedCoins.count(outpt) > 0); @@ -3858,7 +3700,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); it != setLockedCoins.end(); it++) { COutPoint outpt = (*it); @@ -3868,8 +3710,8 @@ void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const /** @} */ // end of Actions -void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t> &mapKeyBirth) const { - AssertLockHeld(cs_wallet); // mapKeyMetadata +void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t>& mapKeyBirth) const { + AssertLockHeld(cs_wallet); mapKeyBirth.clear(); // get birth times for keys with metadata @@ -4085,7 +3927,7 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b if (salvage_wallet) { // Recover readable keypairs: - CWallet dummyWallet(chain, WalletLocation(), WalletDatabase::CreateDummy()); + CWallet dummyWallet(&chain, WalletLocation(), WalletDatabase::CreateDummy()); std::string backup_filename; if (!WalletBatch::Recover(wallet_path, (void *)&dummyWallet, WalletBatch::RecoverKeysOnlyFilter, backup_filename)) { return false; @@ -4097,58 +3939,58 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, uint64_t wallet_creation_flags) { - const std::string& walletFile = location.GetName(); + const std::string& walletFile = WalletDataFilePath(location.GetPath()).string(); // needed to restore wallet transaction meta data after -zapwallettxes std::vector<CWalletTx> vWtx; if (gArgs.GetBoolArg("-zapwallettxes", false)) { - uiInterface.InitMessage(_("Zapping all transactions from wallet...")); + chain.initMessage(_("Zapping all transactions from wallet...")); - std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(chain, location, WalletDatabase::Create(location.GetPath())); + std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, WalletDatabase::Create(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } } - uiInterface.InitMessage(_("Loading wallet...")); + chain.initMessage(_("Loading wallet...")); int64_t nStart = GetTimeMillis(); bool fFirstRun = true; // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - std::shared_ptr<CWallet> walletInstance(new CWallet(chain, location, WalletDatabase::Create(location.GetPath())), ReleaseWallet); + std::shared_ptr<CWallet> walletInstance(new CWallet(&chain, location, WalletDatabase::Create(location.GetPath())), ReleaseWallet); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { - InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect."), + chain.initWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - InitError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); + chain.initError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - InitError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); + chain.initError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); return nullptr; } else { - InitError(strprintf(_("Error loading %s"), walletFile)); + chain.initError(strprintf(_("Error loading %s"), walletFile)); return nullptr; } } - int prev_version = walletInstance->nWalletVersion; + int prev_version = walletInstance->GetVersion(); if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) { int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); @@ -4162,7 +4004,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); if (nMaxVersion < walletInstance->GetVersion()) { - InitError(_("Cannot downgrade wallet")); + chain.initError(_("Cannot downgrade wallet")); return nullptr; } walletInstance->SetMaxVersion(nMaxVersion); @@ -4173,9 +4015,9 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, LOCK(walletInstance->cs_wallet); // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->nWalletVersion; - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >=FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - InitError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); + int max_version = walletInstance->GetVersion(); + if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + chain.initError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); return nullptr; } @@ -4203,7 +4045,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Regenerate the keypool if upgraded to HD if (hd_upgrade) { if (!walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate keys")); + chain.initError(_("Unable to generate keys")); return nullptr; } } @@ -4227,7 +4069,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Top up the keypool if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate initial keys")); + chain.initError(_("Unable to generate initial keys")); return nullptr; } @@ -4235,34 +4077,34 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->ChainStateFlushed(locked_chain->getTipLocator()); } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation - InitError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { LOCK(walletInstance->cs_KeyStore); if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - InitWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); + chain.initWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); } } if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - InitError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); + chain.initError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); return nullptr; } if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - InitError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); + chain.initError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); return nullptr; } if (gArgs.IsArgSet("-mintxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - InitError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); + chain.initError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); return nullptr; } if (n > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-mintxfee") + " " + - _("This is the minimum transaction fee you pay on every transaction.")); + chain.initWarning(AmountHighWarn("-mintxfee") + " " + + _("This is the minimum transaction fee you pay on every transaction.")); } walletInstance->m_min_fee = CFeeRate(n); } @@ -4271,12 +4113,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); + chain.initError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-fallbackfee") + " " + - _("This is the transaction fee you may pay when fee estimates are not available.")); + chain.initWarning(AmountHighWarn("-fallbackfee") + " " + + _("This is the transaction fee you may pay when fee estimates are not available.")); } walletInstance->m_fallback_fee = CFeeRate(nFeePerK); walletInstance->m_allow_fallback_fee = nFeePerK != 0; //disable fallback fee in case value was set to 0, enable if non-null value @@ -4284,32 +4126,55 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-discardfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); + chain.initError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-discardfee") + " " + - _("This is the transaction fee you may discard if change is smaller than dust at this level")); + chain.initWarning(AmountHighWarn("-discardfee") + " " + + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } walletInstance->m_discard_rate = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - InitError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); + chain.initError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-paytxfee") + " " + - _("This is the transaction fee you will pay if you send a transaction.")); + chain.initWarning(AmountHighWarn("-paytxfee") + " " + + _("This is the transaction fee you will pay if you send a transaction.")); } walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); - if (walletInstance->m_pay_tx_fee < ::minRelayTxFee) { - InitError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), - gArgs.GetArg("-paytxfee", ""), ::minRelayTxFee.ToString())); + if (walletInstance->m_pay_tx_fee < chain.relayMinFee()) { + chain.initError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), + gArgs.GetArg("-paytxfee", ""), chain.relayMinFee().ToString())); + return nullptr; + } + } + + if (gArgs.IsArgSet("-maxtxfee")) + { + CAmount nMaxFee = 0; + if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) { + chain.initError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); + return nullptr; + } + if (nMaxFee > HIGH_MAX_TX_FEE) { + chain.initWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + } + if (CFeeRate(nMaxFee, 1000) < chain.relayMinFee()) { + chain.initError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString())); return nullptr; } + walletInstance->m_default_max_tx_fee = nMaxFee; } + + if (chain.relayMinFee().GetFeePerK() > HIGH_TX_FEE_PER_KB) + chain.initWarning(AmountHighWarn("-minrelaytxfee") + " " + + _("The wallet will avoid paying less than the minimum relay fee.")); + walletInstance->m_confirm_target = gArgs.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); walletInstance->m_spend_zero_conf_change = gArgs.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); walletInstance->m_signal_rbf = gArgs.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); @@ -4346,39 +4211,36 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, //We can't rescan beyond non-pruned blocks, stop and throw an error //this might happen if a user uses an old wallet within a pruned node // or if he ran -disablewallet for a longer time, then decided to re-enable - if (fPruneMode) - { + if (chain.getPruneMode()) { int block_height = *tip_height; while (block_height > 0 && locked_chain->haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - InitError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); + chain.initError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); return nullptr; } } - uiInterface.InitMessage(_("Rescanning...")); + chain.initMessage(_("Rescanning...")); walletInstance->WalletLogPrintf("Rescanning last %i blocks (from block %i)...\n", *tip_height - rescan_height, rescan_height); // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) if (walletInstance->nTimeFirstKey) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height)) { + if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } - nStart = GetTimeMillis(); { WalletRescanReserver reserver(walletInstance.get()); if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { - InitError(_("Failed to rescan the wallet during initialization")); + chain.initError(_("Failed to rescan the wallet during initialization")); return nullptr; } } - walletInstance->WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - nStart); walletInstance->ChainStateFlushed(locked_chain->getTipLocator()); walletInstance->database->IncrementUpdateCounter(); @@ -4407,10 +4269,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - uiInterface.LoadWallet(walletInstance); + chain.loadWallet(interfaces::MakeWallet(walletInstance)); - // Register with the validation interface. It's ok to do this after rescan since we're still holding cs_main. - RegisterValidationInterface(walletInstance.get()); + // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. + walletInstance->handleNotifications(); walletInstance->SetBroadcastTransactions(gArgs.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); @@ -4423,11 +4285,22 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return walletInstance; } +void CWallet::handleNotifications() +{ + m_chain_notifications_handler = m_chain->handleNotifications(*this); +} + void CWallet::postInitProcess() { + auto locked_chain = chain().lock(); + LOCK(cs_wallet); + // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(); + ReacceptWalletTransactions(*locked_chain); + + // Update wallet transactions with current mempool transactions. + chain().requestMempoolTransactions(*this); } bool CWallet::BackupWallet(const std::string& strDest) @@ -4470,8 +4343,6 @@ int CMerkleTx::GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const if (hashUnset()) return 0; - AssertLockHeld(cs_main); - return locked_chain.getBlockDepth(hashBlock) * (nIndex == -1 ? -1 : 1); } @@ -4490,17 +4361,14 @@ bool CMerkleTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const return GetBlocksToMaturity(locked_chain) > 0; } -bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state) { - LockAnnotation lock(::cs_main); // Temporary, for AcceptToMemoryPool below. Removed in upcoming commit. - // We must set fInMempool here - while it will be re-set to true by the // entered-mempool callback, if we did not there would be a race where a // user could call sendmoney in a loop and hit spurious out of funds errors // because we think that this newly generated transaction's change is // unavailable as we're not yet aware that it is in the mempool. - bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, - nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + bool ret = locked_chain.submitToMemoryPool(tx, pwallet->m_default_max_tx_fee, state); fInMempool |= ret; return ret; } @@ -4531,7 +4399,7 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; - mempool.GetTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { // Limit output groups to no more than 10 entries, to protect // against inadvertently creating a too-large transaction diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 2a5d6caaf8..510f4b8c04 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -8,18 +8,19 @@ #include <amount.h> #include <interfaces/chain.h> +#include <interfaces/handler.h> #include <outputtype.h> #include <policy/feerate.h> +#include <script/ismine.h> +#include <script/sign.h> #include <streams.h> #include <tinyformat.h> #include <ui_interface.h> #include <util/strencodings.h> -#include <validationinterface.h> -#include <script/ismine.h> -#include <script/sign.h> #include <util/system.h> -#include <wallet/crypter.h> +#include <validationinterface.h> #include <wallet/coinselection.h> +#include <wallet/crypter.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -34,26 +35,6 @@ #include <utility> #include <vector> -//! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -//! This function will perform salvage on the wallet if requested, as long as only one wallet is -//! being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). -bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); - -//! Load wallet databases. -bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); - -//! Complete startup of wallets. -void StartWallets(CScheduler& scheduler); - -//! Flush all wallets in preparation for shutdown. -void FlushWallets(); - -//! Stop all wallets. Wallets will be flushed first. -void StopWallets(); - -//! Close all wallets. -void UnloadWallets(); - //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet clients release the wallet. @@ -92,6 +73,12 @@ static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; static const bool DEFAULT_WALLET_RBF = false; static const bool DEFAULT_WALLETBROADCAST = true; static const bool DEFAULT_DISABLE_WALLET = false; +//! -maxtxfee default +constexpr CAmount DEFAULT_TRANSACTION_MAXFEE{COIN / 10}; +//! Discourage users to set fees higher than this amount (in satoshis) per kB +constexpr CAmount HIGH_TX_FEE_PER_KB{COIN / 100}; +//! -maxtxfee will warn if called with a higher fee than this amount (in satoshis) +constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; //! Pre-calculated constants for input size estimation in *virtual size* static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; @@ -100,8 +87,6 @@ class CCoinControl; class COutput; class CReserveKey; class CScript; -class CTxMemPool; -class CBlockPolicyEstimator; class CWalletTx; struct FeeCalculation; enum class FeeEstimateMode; @@ -390,24 +375,11 @@ public: std::multimap<int64_t, CWalletTx*>::const_iterator m_it_wtxOrdered; // memory only - mutable bool fDebitCached; - mutable bool fCreditCached; - mutable bool fImmatureCreditCached; - mutable bool fAvailableCreditCached; - mutable bool fWatchDebitCached; - mutable bool fWatchCreditCached; - mutable bool fImmatureWatchCreditCached; - mutable bool fAvailableWatchCreditCached; + enum AmountType { DEBIT, CREDIT, IMMATURE_CREDIT, AVAILABLE_CREDIT, AMOUNTTYPE_ENUM_ELEMENTS }; + CAmount GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate = false) const; + mutable CachableAmount m_amounts[AMOUNTTYPE_ENUM_ELEMENTS]; mutable bool fChangeCached; mutable bool fInMempool; - mutable CAmount nDebitCached; - mutable CAmount nCreditCached; - mutable CAmount nImmatureCreditCached; - mutable CAmount nAvailableCreditCached; - mutable CAmount nWatchDebitCached; - mutable CAmount nWatchCreditCached; - mutable CAmount nImmatureWatchCreditCached; - mutable CAmount nAvailableWatchCreditCached; mutable CAmount nChangeCached; CWalletTx(const CWallet* pwalletIn, CTransactionRef arg) : CMerkleTx(std::move(arg)) @@ -424,24 +396,8 @@ public: nTimeReceived = 0; nTimeSmart = 0; fFromMe = false; - fDebitCached = false; - fCreditCached = false; - fImmatureCreditCached = false; - fAvailableCreditCached = false; - fWatchDebitCached = false; - fWatchCreditCached = false; - fImmatureWatchCreditCached = false; - fAvailableWatchCreditCached = false; fChangeCached = false; fInMempool = false; - nDebitCached = 0; - nCreditCached = 0; - nImmatureCreditCached = 0; - nAvailableCreditCached = 0; - nWatchDebitCached = 0; - nWatchCreditCached = 0; - nAvailableWatchCreditCached = 0; - nImmatureWatchCreditCached = 0; nChangeCached = 0; nOrderPos = -1; } @@ -485,14 +441,10 @@ public: //! make sure balances are recalculated void MarkDirty() { - fCreditCached = false; - fAvailableCreditCached = false; - fImmatureCreditCached = false; - fWatchDebitCached = false; - fWatchCreditCached = false; - fAvailableWatchCreditCached = false; - fImmatureWatchCreditCached = false; - fDebitCached = false; + m_amounts[DEBIT].Reset(); + m_amounts[CREDIT].Reset(); + m_amounts[IMMATURE_CREDIT].Reset(); + m_amounts[AVAILABLE_CREDIT].Reset(); fChangeCached = false; } @@ -507,7 +459,7 @@ public: CAmount GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const; CAmount GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true) const; // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct - // annotation "EXCLUSIVE_LOCKS_REQUIRED(cs_main, pwallet->cs_wallet)". The + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid // having to resolve the issue of member access into incomplete type CWallet. CAmount GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true, const isminefilter& filter=ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; @@ -536,11 +488,11 @@ public: int64_t GetTxTime() const; - // RelayWalletTransaction may only be called if fBroadcastTransactions! - bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman); + // Pass this transaction to the node to relay to its peers + bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain); /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state); + bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -639,7 +591,7 @@ class WalletRescanReserver; //forward declarations for ScanForWalletTransactions * A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, * and provides the ability to create new transactions. */ -class CWallet final : public CCryptoKeyStore, public CValidationInterface +class CWallet final : public CCryptoKeyStore, private interfaces::Chain::Notifications { private: std::atomic<bool> fAbortRescan{false}; @@ -650,7 +602,7 @@ private: WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; //! the current wallet version: clients below this version are not able to load the wallet - int nWalletVersion = FEATURE_BASE; + int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; //! the maximum wallet format version: memory-only variable that specifies to what version this wallet may be upgraded int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; @@ -658,6 +610,8 @@ private: int64_t nNextResend = 0; int64_t nLastResend = 0; bool fBroadcastTransactions = false; + // Local time that the tip block was received. Used to schedule wallet rebroadcasts. + std::atomic<int64_t> m_best_block_time {0}; /** * Used to keep track of spent outpoints, and @@ -700,11 +654,11 @@ private: CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::set<int64_t> setInternalKeyPool; + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool; + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; std::atomic<uint64_t> m_wallet_flags{0}; @@ -723,7 +677,7 @@ private: bool AddWatchOnly(const CScript& dest) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Interface for accessing chain state. */ - interfaces::Chain& m_chain; + interfaces::Chain* m_chain; /** Wallet location which includes wallet name (see WalletLocation). */ WalletLocation m_location; @@ -739,7 +693,7 @@ private: * to have seen all transactions in the chain, but is only used to track * live BlockConnected callbacks. */ - uint256 m_last_block_processed; + uint256 m_last_block_processed GUARDED_BY(cs_wallet); public: /* @@ -786,7 +740,10 @@ public: unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ - CWallet(interfaces::Chain& chain, const WalletLocation& location, std::unique_ptr<WalletDatabase> database) : m_chain(chain), m_location(location), database(std::move(database)) + CWallet(interfaces::Chain* chain, const WalletLocation& location, std::unique_ptr<WalletDatabase> database) + : m_chain(chain), + m_location(location), + database(std::move(database)) { } @@ -810,8 +767,14 @@ public: std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); + /** Registered interfaces::Chain::Notifications handler. */ + std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; + + /** Register the wallet for chain notifications */ + void handleNotifications(); + /** Interface for accessing chain state. */ - interfaces::Chain& chain() const { return m_chain; } + interfaces::Chain& chain() const { assert(m_chain); return *m_chain; } const CWalletTx* GetWalletTx(const uint256& hash) const; @@ -922,8 +885,9 @@ public: bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); void LoadToWallet(const CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void TransactionAddedToMempool(const CTransactionRef& tx) override; - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const CBlock& block) override; + void UpdatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); struct ScanResult { @@ -943,16 +907,17 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(); - void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override EXCLUSIVE_LOCKS_REQUIRED(cs_main); - // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! - std::vector<uint256> ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman); - CAmount GetBalance(const isminefilter& filter=ISMINE_SPENDABLE, const int min_depth=0) const; - CAmount GetUnconfirmedBalance() const; - CAmount GetImmatureBalance() const; - CAmount GetUnconfirmedWatchOnlyBalance() const; - CAmount GetImmatureWatchOnlyBalance() const; - CAmount GetLegacyBalance(const isminefilter& filter, int minDepth) const; + void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ResendWalletTransactions(); + struct Balance { + CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more + CAmount m_mine_untrusted_pending{0}; //!< Untrusted, but in mempool (pending) + CAmount m_mine_immature{0}; //!< Immature coinbases in the main chain + CAmount m_watchonly_trusted{0}; + CAmount m_watchonly_untrusted_pending{0}; + CAmount m_watchonly_immature{0}; + }; + Balance GetBalance(int min_depth = 0) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); @@ -971,7 +936,7 @@ public: */ bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); - bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state); + bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state); bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const { @@ -997,6 +962,8 @@ public: CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; OutputType m_default_change_type{DEFAULT_CHANGE_TYPE}; + /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ + CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -1065,11 +1032,9 @@ public: const std::string& GetLabelName(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void GetScriptForMining(std::shared_ptr<CReserveScript> &script); - unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { - AssertLockHeld(cs_wallet); // set{Ex,In}ternalKeyPool + AssertLockHeld(cs_wallet); return setInternalKeyPool.size() + setExternalKeyPool.size(); } @@ -1226,8 +1191,14 @@ public: bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info); }; +/** + * Called periodically by the schedule thread. Prompts individual wallets to resend + * their transactions. Actual rebroadcast schedule is managed by the wallets themselves. + */ +void MaybeResendWalletTxs(); + /** A key allocated from the key pool. */ -class CReserveKey final : public CReserveScript +class CReserveKey { protected: CWallet* pwallet; @@ -1252,7 +1223,6 @@ public: void ReturnKey(); bool GetReservedKey(CPubKey &pubkey, bool internal = false); void KeepKey(); - void KeepScript() override { KeepKey(); } }; /** RAII object to check and reserve a wallet rescan */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 2783f83fd6..3122cd6fa4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -5,7 +5,7 @@ #include <wallet/walletdb.h> -#include <consensus/tx_verify.h> +#include <consensus/tx_check.h> #include <consensus/validation.h> #include <fs.h> #include <key_io.h> @@ -422,8 +422,15 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strType != "minversion" && strType != "acentry") { wss.m_unknown_records++; } - } catch (...) - { + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } catch (...) { + if (strErr.empty()) { + strErr = "Caught unknown exception in ReadKeyValue"; + } return false; } return true; diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 797f051189..1ff1e8b840 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -4,7 +4,6 @@ #include <base58.h> #include <fs.h> -#include <interfaces/chain.h> #include <util/system.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -28,8 +27,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: return nullptr; } // dummy chain interface - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet_instance(new CWallet(*chain, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); if (load_wallet_ret != DBErrors::LOAD_OK) { @@ -56,8 +54,7 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa } // dummy chain interface - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet_instance(new CWallet(*chain, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); DBErrors load_wallet_ret; try { bool first_run; diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index d779251d56..b227a15556 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -4,6 +4,7 @@ #include <wallet/walletutil.h> +#include <logging.h> #include <util/system.h> fs::path GetWalletDir() @@ -30,10 +31,14 @@ fs::path GetWalletDir() static bool IsBerkeleyBtree(const fs::path& path) { + if (!fs::exists(path)) return false; + // A Berkeley DB Btree file has at least 4K. // This check also prevents opening lock files. boost::system::error_code ec; - if (fs::file_size(path, ec) < 4096) return false; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 4096) return false; fsbridge::ifstream file(path, std::ios::binary); if (!file.is_open()) return false; @@ -54,8 +59,14 @@ std::vector<fs::path> ListWalletDir() const fs::path wallet_dir = GetWalletDir(); const size_t offset = wallet_dir.string().size() + 1; std::vector<fs::path> paths; + boost::system::error_code ec; + + for (auto it = fs::recursive_directory_iterator(wallet_dir, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (ec) { + LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); + continue; + } - for (auto it = fs::recursive_directory_iterator(wallet_dir); it != fs::recursive_directory_iterator(); ++it) { // Get wallet path relative to walletdir by removing walletdir from the wallet path. // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. const fs::path path = it->path().string().substr(offset); diff --git a/test/config.ini.in b/test/config.ini.in index 6b7ef70659..060c553da2 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -6,6 +6,7 @@ # test/functional/test_runner.py and test/util/bitcoin-util-test.py [environment] +PACKAGE_NAME=@PACKAGE_NAME@ SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ EXEEXT=@EXEEXT@ diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 5bb3b5c094..45ecaabe14 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -11,6 +11,7 @@ from collections import defaultdict, namedtuple import heapq import itertools import os +import pathlib import re import sys import tempfile @@ -51,9 +52,23 @@ def main(): if not args.testdir: print("Opening latest test directory: {}".format(testdir), file=sys.stderr) + colors = defaultdict(lambda: '') + if args.color: + colors["test"] = "\033[0;36m" # CYAN + colors["node0"] = "\033[0;34m" # BLUE + colors["node1"] = "\033[0;32m" # GREEN + colors["node2"] = "\033[0;31m" # RED + colors["node3"] = "\033[0;33m" # YELLOW + colors["reset"] = "\033[0m" # Reset font color + log_events = read_logs(testdir) - print_logs(log_events, color=args.color, html=args.html) + if args.html: + print_logs_html(log_events) + else: + print_logs_plain(log_events, colors) + print_node_warnings(testdir, colors) + def read_logs(tmp_dir): """Reads log files. @@ -71,6 +86,26 @@ def read_logs(tmp_dir): return heapq.merge(*[get_log_events(source, f) for source, f in files]) +def print_node_warnings(tmp_dir, colors): + """Print nodes' errors and warnings""" + + warnings = [] + for stream in ['stdout', 'stderr']: + for i in itertools.count(): + folder = "{}/node{}/{}".format(tmp_dir, i, stream) + if not os.path.isdir(folder): + break + for (_, _, fns) in os.walk(folder): + for fn in fns: + warning = pathlib.Path('{}/{}'.format(folder, fn)).read_text().strip() + if warning: + warnings.append(("node{} {}".format(i, stream), warning)) + + print() + for w in warnings: + print("{} {} {} {}".format(colors[w[0].split()[0]], w[0], w[1], colors["reset"])) + + def find_latest_test_dir(): """Returns the latest tmpfile test directory prefix.""" tmpdir = tempfile.gettempdir() @@ -127,18 +162,9 @@ def get_log_events(source, logfile): except FileNotFoundError: print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr) -def print_logs(log_events, color=False, html=False): - """Renders the iterator of log events into text or html.""" - if not html: - colors = defaultdict(lambda: '') - if color: - colors["test"] = "\033[0;36m" # CYAN - colors["node0"] = "\033[0;34m" # BLUE - colors["node1"] = "\033[0;32m" # GREEN - colors["node2"] = "\033[0;31m" # RED - colors["node3"] = "\033[0;33m" # YELLOW - colors["reset"] = "\033[0m" # Reset font color +def print_logs_plain(log_events, colors): + """Renders the iterator of log events into text.""" for event in log_events: lines = event.event.splitlines() print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, lines[0], colors["reset"])) @@ -146,7 +172,9 @@ def print_logs(log_events, color=False, html=False): for line in lines[1:]: print("{0}{1}{2}".format(colors[event.source.rstrip()], line, colors["reset"])) - else: + +def print_logs_html(log_events): + """Renders the iterator of log events into html.""" try: import jinja2 except ImportError: diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 02deae92f3..454eb583f7 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -58,7 +58,7 @@ class BadTxTemplate: class OutputMissing(BadTxTemplate): reject_reason = "bad-txns-vout-empty" - expect_disconnect = False + expect_disconnect = True def get_tx(self): tx = CTransaction() @@ -69,11 +69,15 @@ class OutputMissing(BadTxTemplate): class InputMissing(BadTxTemplate): reject_reason = "bad-txns-vin-empty" - expect_disconnect = False + expect_disconnect = True + # We use a blank transaction here to make sure + # it is interpreted as a non-witness transaction. + # Otherwise the transaction will fail the + # "surpufluous witness" check during deserialization + # rather than the input count check. def get_tx(self): tx = CTransaction() - tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE] * 100))) tx.calc_sha256() return tx diff --git a/test/functional/example_test.py b/test/functional/example_test.py index f367e4fca8..a2726763d0 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test @@ -117,7 +117,7 @@ class ExampleTest(BitcoinTestFramework): # sync_all() should not include node2, since we're not expecting it to # sync. connect_nodes(self.nodes[0], 1) - self.sync_all([self.nodes[0:2]]) + self.sync_all(self.nodes[0:2]) # Use setup_nodes() to customize the node start behaviour (for example if # you don't want to start all nodes at the start of the test). @@ -141,7 +141,7 @@ class ExampleTest(BitcoinTestFramework): # Generating a block on one of the nodes will get us out of IBD blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] - self.sync_all([self.nodes[0:2]]) + self.sync_all(self.nodes[0:2]) # Notice above how we called an RPC by calling a method with the same # name on the node object. Notice also how we used a keyword argument diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 0eb3dd440b..b7814bf33e 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -32,7 +32,7 @@ Start three nodes: import time from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.key import CECKey +from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, COutPoint, @@ -104,9 +104,9 @@ class AssumeValidTest(BitcoinTestFramework): self.blocks = [] # Get a pubkey for the coinbase TXO - coinbase_key = CECKey() - coinbase_key.set_secretbytes(b"horsebattery") - coinbase_pubkey = coinbase_key.get_pubkey() + coinbase_key = ECKey() + coinbase_key.generate() + coinbase_pubkey = coinbase_key.get_pubkey().get_bytes() # Create the first block with a coinbase output to our key height = 1 diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8466f851ca..fdb60fb0e8 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -10,7 +10,13 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, ToHex from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, get_bip9_status, satoshi_round, sync_blocks +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + get_bip9_status, + satoshi_round, +) SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -63,7 +69,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC utxos = self.nodes[0].listunspent(0, 0) - assert(len(utxos) > 0) + assert len(utxos) > 0 utxo = utxos[0] @@ -253,7 +259,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].generate(1) cur_time += 600 - assert(tx2.hash in self.nodes[0].getrawmempool()) + assert tx2.hash in self.nodes[0].getrawmempool() test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=True) test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) @@ -264,23 +270,23 @@ class BIP68Test(BitcoinTestFramework): # Advance the time on the node so that we can test timelocks self.nodes[0].setmocktime(cur_time+600) self.nodes[0].generate(1) - assert(tx2.hash not in self.nodes[0].getrawmempool()) + assert tx2.hash not in self.nodes[0].getrawmempool() # Now that tx2 is not in the mempool, a sequence locked spend should # succeed tx3 = test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx3.hash in self.nodes[0].getrawmempool() self.nodes[0].generate(1) - assert(tx3.hash not in self.nodes[0].getrawmempool()) + assert tx3.hash not in self.nodes[0].getrawmempool() # One more test, this time using height locks tx4 = test_nonzero_locks(tx3, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx4.hash in self.nodes[0].getrawmempool()) + assert tx4.hash in self.nodes[0].getrawmempool() # Now try combining confirmed and unconfirmed inputs tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx5.hash not in self.nodes[0].getrawmempool()) + assert tx5.hash not in self.nodes[0].getrawmempool() utxos = self.nodes[0].listunspent() tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) @@ -299,8 +305,8 @@ class BIP68Test(BitcoinTestFramework): # If we invalidate the tip, tx3 should get added to the mempool, causing # tx4 to be removed (fails sequence-lock). self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - assert(tx4.hash not in self.nodes[0].getrawmempool()) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx4.hash not in self.nodes[0].getrawmempool() + assert tx3.hash in self.nodes[0].getrawmempool() # Now mine 2 empty blocks to reorg out the current tip (labeled tip-1 in # diagram above). @@ -319,8 +325,8 @@ class BIP68Test(BitcoinTestFramework): cur_time += 1 mempool = self.nodes[0].getrawmempool() - assert(tx3.hash not in mempool) - assert(tx2.hash in mempool) + assert tx3.hash not in mempool + assert tx2.hash in mempool # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) @@ -332,7 +338,7 @@ class BIP68Test(BitcoinTestFramework): # being run, then it's possible the test has activated the soft fork, and # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): - assert(get_bip9_status(self.nodes[0], 'csv')['status'] != 'active') + assert get_bip9_status(self.nodes[0], 'csv')['status'] != 'active' txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) tx1 = FromHex(CTransaction(), self.nodes[0].getrawtransaction(txid)) @@ -372,7 +378,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): @@ -385,7 +391,7 @@ class BIP68Test(BitcoinTestFramework): assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], "locked_in") self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], "active") - sync_blocks(self.nodes) + self.sync_blocks() # Use self.nodes[1] to test that version 2 transactions are standard. def test_version2_relay(self): diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 5253ff7aaa..3ad83cd2b3 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -14,7 +14,7 @@ from test_framework.blocktools import ( get_legacy_sigopcount_block, MAX_BLOCK_SIGOPS, ) -from test_framework.key import CECKey +from test_framework.key import ECKey from test_framework.messages import ( CBlock, COIN, @@ -86,9 +86,9 @@ class FullBlockTest(BitcoinTestFramework): self.bootstrap_p2p() # Add one p2p connection to the node self.block_heights = {} - self.coinbase_key = CECKey() - self.coinbase_key.set_secretbytes(b"horsebattery") - self.coinbase_pubkey = self.coinbase_key.get_pubkey() + self.coinbase_key = ECKey() + self.coinbase_key.generate() + self.coinbase_pubkey = self.coinbase_key.get_pubkey().get_bytes() self.tip = None self.blocks = {} self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) @@ -98,7 +98,7 @@ class FullBlockTest(BitcoinTestFramework): # Create a new block b0 = self.next_block(0) self.save_spendable_output() - self.sync_blocks([b0]) + self.send_blocks([b0]) # These constants chosen specifically to trigger an immature coinbase spend # at a certain time below. @@ -110,7 +110,7 @@ class FullBlockTest(BitcoinTestFramework): for i in range(NUM_BUFFER_BLOCKS_TO_GENERATE): blocks.append(self.next_block("maturitybuffer.{}".format(i))) self.save_spendable_output() - self.sync_blocks(blocks) + self.send_blocks(blocks) # collect spendable outputs now to avoid cluttering the code later on out = [] @@ -126,7 +126,7 @@ class FullBlockTest(BitcoinTestFramework): b2 = self.next_block(2, spend=out[1]) self.save_spendable_output() - self.sync_blocks([b1, b2], timeout=4) + self.send_blocks([b1, b2], timeout=4) # Select a txn with an output eligible for spending. This won't actually be spent, # since we're testing submission of a series of blocks with invalid txns. @@ -146,23 +146,9 @@ class FullBlockTest(BitcoinTestFramework): badtx = template.get_tx() if TxTemplate != invalid_txs.InputMissing: self.sign_tx(badtx, attempt_spend_tx) - else: - # Segwit is active in regtest at this point, so to deserialize a - # transaction without any inputs correctly, we set the outputs - # to an empty list. This is a hack, as the serialization of an - # empty list of outputs is deserialized as flags==0 and thus - # deserialization of the outputs is skipped. - # A policy check requires "loose" txs to be of a minimum size, - # so vtx is not set to be empty in the TxTemplate class and we - # only apply the workaround where txs are not "loose", i.e. in - # blocks. - # - # The workaround has the purpose that both sides calculate - # the same tx hash in the merkle tree - badtx.vout = [] badtx.rehash() badblock = self.update_block(blockname, [badtx]) - self.sync_blocks( + self.send_blocks( [badblock], success=False, reject_reason=(template.block_reject_reason or template.reject_reason), reconnect=True, timeout=2) @@ -179,7 +165,7 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(1) b3 = self.next_block(3, spend=out[1]) txout_b3 = b3.vtx[1] - self.sync_blocks([b3], False) + self.send_blocks([b3], False) # Now we add another block to make the alternative chain longer. # @@ -187,7 +173,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b3 (1) -> b4 (2) self.log.info("Reorg to a longer chain") b4 = self.next_block(4, spend=out[2]) - self.sync_blocks([b4]) + self.send_blocks([b4]) # ... and back to the first chain. # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -195,11 +181,11 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(2) b5 = self.next_block(5, spend=out[2]) self.save_spendable_output() - self.sync_blocks([b5], False) + self.send_blocks([b5], False) self.log.info("Reorg back to the original chain") b6 = self.next_block(6, spend=out[3]) - self.sync_blocks([b6], True) + self.send_blocks([b6], True) # Try to create a fork that double-spends # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -208,10 +194,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a chain with a double spend, even if it is longer") self.move_tip(5) b7 = self.next_block(7, spend=out[2]) - self.sync_blocks([b7], False) + self.send_blocks([b7], False) b8 = self.next_block(8, spend=out[4]) - self.sync_blocks([b8], False, reconnect=True) + self.send_blocks([b8], False, reconnect=True) # Try to create a block that has too much fee # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -220,7 +206,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block where the miner creates too much coinbase reward") self.move_tip(6) b9 = self.next_block(9, spend=out[4], additional_coinbase_value=1) - self.sync_blocks([b9], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b9], success=False, reject_reason='bad-cb-amount', reconnect=True) # Create a fork that ends in a block with too much fee (the one that causes the reorg) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -229,10 +215,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer") self.move_tip(5) b10 = self.next_block(10, spend=out[3]) - self.sync_blocks([b10], False) + self.send_blocks([b10], False) b11 = self.next_block(11, spend=out[4], additional_coinbase_value=1) - self.sync_blocks([b11], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b11], success=False, reject_reason='bad-cb-amount', reconnect=True) # Try again, but with a valid fork first # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -245,7 +231,7 @@ class FullBlockTest(BitcoinTestFramework): b13 = self.next_block(13, spend=out[4]) self.save_spendable_output() b14 = self.next_block(14, spend=out[5], additional_coinbase_value=1) - self.sync_blocks([b12, b13, b14], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b12, b13, b14], success=False, reject_reason='bad-cb-amount', reconnect=True) # New tip should be b13. assert_equal(node.getbestblockhash(), b13.hash) @@ -259,12 +245,12 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(13) b15 = self.next_block(15, spend=out[5], script=lots_of_checksigs) self.save_spendable_output() - self.sync_blocks([b15], True) + self.send_blocks([b15], True) self.log.info("Reject a block with too many checksigs") too_many_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) b16 = self.next_block(16, spend=out[6], script=too_many_checksigs) - self.sync_blocks([b16], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b16], success=False, reject_reason='bad-blk-sigops', reconnect=True) # Attempt to spend a transaction created on a different fork # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -273,7 +259,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block with a spend from a re-org'ed out tx") self.move_tip(15) b17 = self.next_block(17, spend=txout_b3) - self.sync_blocks([b17], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b17], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a transaction created on a different fork (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -283,10 +269,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block with a spend from a re-org'ed out tx (on a forked chain)") self.move_tip(13) b18 = self.next_block(18, spend=txout_b3) - self.sync_blocks([b18], False) + self.send_blocks([b18], False) b19 = self.next_block(19, spend=out[6]) - self.sync_blocks([b19], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b19], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a coinbase at depth too low # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -295,7 +281,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block spending an immature coinbase.") self.move_tip(15) b20 = self.next_block(20, spend=out[7]) - self.sync_blocks([b20], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') + self.send_blocks([b20], success=False, reject_reason='bad-txns-premature-spend-of-coinbase', reconnect=True) # Attempt to spend a coinbase at depth too low (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -305,10 +291,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block spending an immature coinbase (on a forked chain)") self.move_tip(13) b21 = self.next_block(21, spend=out[6]) - self.sync_blocks([b21], False) + self.send_blocks([b21], False) b22 = self.next_block(22, spend=out[5]) - self.sync_blocks([b22], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') + self.send_blocks([b22], success=False, reject_reason='bad-txns-premature-spend-of-coinbase', reconnect=True) # Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -326,7 +312,7 @@ class FullBlockTest(BitcoinTestFramework): b23 = self.update_block(23, [tx]) # Make sure the math above worked out to produce a max-sized block assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE) - self.sync_blocks([b23], True) + self.send_blocks([b23], True) self.save_spendable_output() self.log.info("Reject a block of size MAX_BLOCK_BASE_SIZE + 1") @@ -337,10 +323,10 @@ class FullBlockTest(BitcoinTestFramework): tx.vout = [CTxOut(0, script_output)] b24 = self.update_block(24, [tx]) assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE + 1) - self.sync_blocks([b24], success=False, reject_reason='bad-blk-length', reconnect=True) + self.send_blocks([b24], success=False, reject_reason='bad-blk-length', reconnect=True) b25 = self.next_block(25, spend=out[7]) - self.sync_blocks([b25], False) + self.send_blocks([b25], False) # Create blocks with a coinbase input script size out of range # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -355,11 +341,11 @@ class FullBlockTest(BitcoinTestFramework): # update_block causes the merkle root to get updated, even with no new # transactions, and updates the required state. b26 = self.update_block(26, []) - self.sync_blocks([b26], success=False, reject_reason='bad-cb-length', reconnect=True) + self.send_blocks([b26], success=False, reject_reason='bad-cb-length', reconnect=True) # Extend the b26 chain to make sure bitcoind isn't accepting b26 b27 = self.next_block(27, spend=out[7]) - self.sync_blocks([b27], False) + self.send_blocks([b27], False) # Now try a too-large-coinbase script self.move_tip(15) @@ -367,11 +353,11 @@ class FullBlockTest(BitcoinTestFramework): b28.vtx[0].vin[0].scriptSig = b'\x00' * 101 b28.vtx[0].rehash() b28 = self.update_block(28, []) - self.sync_blocks([b28], success=False, reject_reason='bad-cb-length', reconnect=True) + self.send_blocks([b28], success=False, reject_reason='bad-cb-length', reconnect=True) # Extend the b28 chain to make sure bitcoind isn't accepting b28 b29 = self.next_block(29, spend=out[7]) - self.sync_blocks([b29], False) + self.send_blocks([b29], False) # b30 has a max-sized coinbase scriptSig. self.move_tip(23) @@ -379,7 +365,7 @@ class FullBlockTest(BitcoinTestFramework): b30.vtx[0].vin[0].scriptSig = b'\x00' * 100 b30.vtx[0].rehash() b30 = self.update_block(30, []) - self.sync_blocks([b30], True) + self.send_blocks([b30], True) self.save_spendable_output() # b31 - b35 - check sigops of OP_CHECKMULTISIG / OP_CHECKMULTISIGVERIFY / OP_CHECKSIGVERIFY @@ -395,7 +381,7 @@ class FullBlockTest(BitcoinTestFramework): lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) b31 = self.next_block(31, spend=out[8], script=lots_of_multisigs) assert_equal(get_legacy_sigopcount_block(b31), MAX_BLOCK_SIGOPS) - self.sync_blocks([b31], True) + self.send_blocks([b31], True) self.save_spendable_output() # this goes over the limit because the coinbase has one sigop @@ -403,33 +389,33 @@ class FullBlockTest(BitcoinTestFramework): too_many_multisigs = CScript([OP_CHECKMULTISIG] * (MAX_BLOCK_SIGOPS // 20)) b32 = self.next_block(32, spend=out[9], script=too_many_multisigs) assert_equal(get_legacy_sigopcount_block(b32), MAX_BLOCK_SIGOPS + 1) - self.sync_blocks([b32], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b32], success=False, reject_reason='bad-blk-sigops', reconnect=True) # CHECKMULTISIGVERIFY self.log.info("Accept a block with the max number of OP_CHECKMULTISIGVERIFY sigops") self.move_tip(31) lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) b33 = self.next_block(33, spend=out[9], script=lots_of_multisigs) - self.sync_blocks([b33], True) + self.send_blocks([b33], True) self.save_spendable_output() self.log.info("Reject a block with too many OP_CHECKMULTISIGVERIFY sigops") too_many_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * (MAX_BLOCK_SIGOPS // 20)) b34 = self.next_block(34, spend=out[10], script=too_many_multisigs) - self.sync_blocks([b34], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b34], success=False, reject_reason='bad-blk-sigops', reconnect=True) # CHECKSIGVERIFY self.log.info("Accept a block with the max number of OP_CHECKSIGVERIFY sigops") self.move_tip(33) lots_of_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS - 1)) b35 = self.next_block(35, spend=out[10], script=lots_of_checksigs) - self.sync_blocks([b35], True) + self.send_blocks([b35], True) self.save_spendable_output() self.log.info("Reject a block with too many OP_CHECKSIGVERIFY sigops") too_many_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS)) b36 = self.next_block(36, spend=out[11], script=too_many_checksigs) - self.sync_blocks([b36], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b36], success=False, reject_reason='bad-blk-sigops', reconnect=True) # Check spending of a transaction in a block which failed to connect # @@ -446,12 +432,12 @@ class FullBlockTest(BitcoinTestFramework): txout_b37 = b37.vtx[1] tx = self.create_and_sign_transaction(out[11], 0) b37 = self.update_block(37, [tx]) - self.sync_blocks([b37], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b37], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # attempt to spend b37's first non-coinbase tx, at which point b37 was still considered valid self.move_tip(35) b38 = self.next_block(38, spend=txout_b37) - self.sync_blocks([b38], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b38], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Check P2SH SigOp counting # @@ -501,7 +487,7 @@ class FullBlockTest(BitcoinTestFramework): b39_outputs += 1 b39 = self.update_block(39, []) - self.sync_blocks([b39], True) + self.send_blocks([b39], True) self.save_spendable_output() # Test sigops in P2SH redeem scripts @@ -528,7 +514,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b39.vtx[i].sha256, 0), b'')) # Note: must pass the redeem_script (not p2sh_script) to the signature hash function (sighash, err) = SignatureHash(redeem_script, tx, 1, SIGHASH_ALL) - sig = self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL])) + sig = self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL])) scriptSig = CScript([sig, redeem_script]) tx.vin[1].scriptSig = scriptSig @@ -543,7 +529,7 @@ class FullBlockTest(BitcoinTestFramework): tx.rehash() new_txs.append(tx) self.update_block(40, new_txs) - self.sync_blocks([b40], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b40], success=False, reject_reason='bad-blk-sigops', reconnect=True) # same as b40, but one less sigop self.log.info("Accept a block with the max number of P2SH sigops") @@ -556,7 +542,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b41_sigops_to_fill))) tx.rehash() self.update_block(41, [tx]) - self.sync_blocks([b41], True) + self.send_blocks([b41], True) # Fork off of b39 to create a constant base again # @@ -569,7 +555,7 @@ class FullBlockTest(BitcoinTestFramework): b43 = self.next_block(43, spend=out[13]) self.save_spendable_output() - self.sync_blocks([b42, b43], True) + self.send_blocks([b42, b43], True) # Test a number of really invalid scenarios # @@ -591,7 +577,7 @@ class FullBlockTest(BitcoinTestFramework): self.tip = b44 self.block_heights[b44.sha256] = height self.blocks[44] = b44 - self.sync_blocks([b44], True) + self.send_blocks([b44], True) self.log.info("Reject a block with a non-coinbase as the first tx") non_coinbase = self.create_tx(out[15], 0, 1) @@ -606,7 +592,7 @@ class FullBlockTest(BitcoinTestFramework): self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256] + 1 self.tip = b45 self.blocks[45] = b45 - self.sync_blocks([b45], success=False, reject_reason='bad-cb-missing', reconnect=True) + self.send_blocks([b45], success=False, reject_reason='bad-cb-missing', reconnect=True) self.log.info("Reject a block with no transactions") self.move_tip(44) @@ -621,7 +607,7 @@ class FullBlockTest(BitcoinTestFramework): self.tip = b46 assert 46 not in self.blocks self.blocks[46] = b46 - self.sync_blocks([b46], success=False, reject_reason='bad-blk-length', reconnect=True) + self.send_blocks([b46], success=False, reject_reason='bad-blk-length', reconnect=True) self.log.info("Reject a block with invalid work") self.move_tip(44) @@ -630,35 +616,35 @@ class FullBlockTest(BitcoinTestFramework): while b47.sha256 < target: b47.nNonce += 1 b47.rehash() - self.sync_blocks([b47], False, force_send=True, reject_reason='high-hash') + self.send_blocks([b47], False, force_send=True, reject_reason='high-hash', reconnect=True) self.log.info("Reject a block with a timestamp >2 hours in the future") self.move_tip(44) b48 = self.next_block(48, solve=False) b48.nTime = int(time.time()) + 60 * 60 * 3 b48.solve() - self.sync_blocks([b48], False, force_send=True, reject_reason='time-too-new') + self.send_blocks([b48], False, force_send=True, reject_reason='time-too-new') self.log.info("Reject a block with invalid merkle hash") self.move_tip(44) b49 = self.next_block(49) b49.hashMerkleRoot += 1 b49.solve() - self.sync_blocks([b49], success=False, reject_reason='bad-txnmrklroot', reconnect=True) + self.send_blocks([b49], success=False, reject_reason='bad-txnmrklroot', reconnect=True) self.log.info("Reject a block with incorrect POW limit") self.move_tip(44) b50 = self.next_block(50) b50.nBits = b50.nBits - 1 b50.solve() - self.sync_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True) + self.send_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True) self.log.info("Reject a block with two coinbase transactions") self.move_tip(44) b51 = self.next_block(51) cb2 = create_coinbase(51, self.coinbase_pubkey) b51 = self.update_block(51, [cb2]) - self.sync_blocks([b51], success=False, reject_reason='bad-cb-multiple', reconnect=True) + self.send_blocks([b51], success=False, reject_reason='bad-cb-multiple', reconnect=True) self.log.info("Reject a block with duplicate transactions") # Note: txns have to be in the right position in the merkle tree to trigger this error @@ -666,7 +652,7 @@ class FullBlockTest(BitcoinTestFramework): b52 = self.next_block(52, spend=out[15]) tx = self.create_tx(b52.vtx[1], 0, 1) b52 = self.update_block(52, [tx, tx]) - self.sync_blocks([b52], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b52], success=False, reject_reason='bad-txns-duplicate', reconnect=True) # Test block timestamps # -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) @@ -674,21 +660,21 @@ class FullBlockTest(BitcoinTestFramework): # self.move_tip(43) b53 = self.next_block(53, spend=out[14]) - self.sync_blocks([b53], False) + self.send_blocks([b53], False) self.save_spendable_output() self.log.info("Reject a block with timestamp before MedianTimePast") b54 = self.next_block(54, spend=out[15]) b54.nTime = b35.nTime - 1 b54.solve() - self.sync_blocks([b54], False, force_send=True, reject_reason='time-too-old') + self.send_blocks([b54], False, force_send=True, reject_reason='time-too-old', reconnect=True) # valid timestamp self.move_tip(53) b55 = self.next_block(55, spend=out[15]) b55.nTime = b35.nTime self.update_block(55, []) - self.sync_blocks([b55], True) + self.send_blocks([b55], True) self.save_spendable_output() # Test Merkle tree malleability @@ -733,7 +719,7 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(len(b56.vtx), 3) b56 = self.update_block(56, [tx1]) assert_equal(b56.hash, b57.hash) - self.sync_blocks([b56], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b56], success=False, reject_reason='bad-txns-duplicate', reconnect=True) # b57p2 - a good block with 6 tx'es, don't submit until end self.move_tip(55) @@ -753,13 +739,13 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(b56p2.hash, b57p2.hash) assert_equal(len(b56p2.vtx), 6) b56p2 = self.update_block("b56p2", [tx3, tx4]) - self.sync_blocks([b56p2], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b56p2], success=False, reject_reason='bad-txns-duplicate', reconnect=True) self.move_tip("57p2") - self.sync_blocks([b57p2], True) + self.send_blocks([b57p2], True) self.move_tip(57) - self.sync_blocks([b57], False) # The tip is not updated because 57p2 seen first + self.send_blocks([b57], False) # The tip is not updated because 57p2 seen first self.save_spendable_output() # Test a few invalid tx types @@ -773,12 +759,12 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(57) b58 = self.next_block(58, spend=out[17]) tx = CTransaction() - assert(len(out[17].vout) < 42) + assert len(out[17].vout) < 42 tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), 0xffffffff)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() b58 = self.update_block(58, [tx]) - self.sync_blocks([b58], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b58], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # tx with output value > input value self.log.info("Reject a block with a transaction with outputs > inputs") @@ -786,12 +772,12 @@ class FullBlockTest(BitcoinTestFramework): b59 = self.next_block(59) tx = self.create_and_sign_transaction(out[17], 51 * COIN) b59 = self.update_block(59, [tx]) - self.sync_blocks([b59], success=False, reject_reason='bad-txns-in-belowout', reconnect=True) + self.send_blocks([b59], success=False, reject_reason='bad-txns-in-belowout', reconnect=True) # reset to good chain self.move_tip(57) b60 = self.next_block(60, spend=out[17]) - self.sync_blocks([b60], True) + self.send_blocks([b60], True) self.save_spendable_output() # Test BIP30 @@ -810,7 +796,7 @@ class FullBlockTest(BitcoinTestFramework): b61.vtx[0].rehash() b61 = self.update_block(61, []) assert_equal(b60.vtx[0].serialize(), b61.vtx[0].serialize()) - self.sync_blocks([b61], success=False, reject_reason='bad-txns-BIP30', reconnect=True) + self.send_blocks([b61], success=False, reject_reason='bad-txns-BIP30', reconnect=True) # Test tx.isFinal is properly rejected (not an exhaustive tx.isFinal test, that should be in data-driven transaction tests) # @@ -824,10 +810,10 @@ class FullBlockTest(BitcoinTestFramework): tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) - assert(tx.vin[0].nSequence < 0xffffffff) + assert tx.vin[0].nSequence < 0xffffffff tx.calc_sha256() b62 = self.update_block(62, [tx]) - self.sync_blocks([b62], success=False, reject_reason='bad-txns-nonfinal') + self.send_blocks([b62], success=False, reject_reason='bad-txns-nonfinal', reconnect=True) # Test a non-final coinbase is also rejected # @@ -841,7 +827,7 @@ class FullBlockTest(BitcoinTestFramework): b63.vtx[0].vin[0].nSequence = 0xDEADBEEF b63.vtx[0].rehash() b63 = self.update_block(63, []) - self.sync_blocks([b63], success=False, reject_reason='bad-txns-nonfinal') + self.send_blocks([b63], success=False, reject_reason='bad-txns-nonfinal', reconnect=True) # This checks that a block with a bloated VARINT between the block_header and the array of tx such that # the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint, @@ -875,7 +861,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) b64a = self.update_block("64a", [tx]) assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) - self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') + self.send_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently # resend the header message, it won't send us the getdata message again. Just @@ -891,7 +877,7 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE) self.blocks[64] = b64 b64 = self.update_block(64, []) - self.sync_blocks([b64], True) + self.send_blocks([b64], True) self.save_spendable_output() # Spend an output created in the block itself @@ -904,7 +890,7 @@ class FullBlockTest(BitcoinTestFramework): tx1 = self.create_and_sign_transaction(out[19], out[19].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 0) b65 = self.update_block(65, [tx1, tx2]) - self.sync_blocks([b65], True) + self.send_blocks([b65], True) self.save_spendable_output() # Attempt to spend an output created later in the same block @@ -917,7 +903,7 @@ class FullBlockTest(BitcoinTestFramework): tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 1) b66 = self.update_block(66, [tx2, tx1]) - self.sync_blocks([b66], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b66], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to double-spend a transaction created in a block # @@ -932,7 +918,7 @@ class FullBlockTest(BitcoinTestFramework): tx2 = self.create_and_sign_transaction(tx1, 1) tx3 = self.create_and_sign_transaction(tx1, 2) b67 = self.update_block(67, [tx1, tx2, tx3]) - self.sync_blocks([b67], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b67], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # More tests of block subsidy # @@ -951,14 +937,14 @@ class FullBlockTest(BitcoinTestFramework): b68 = self.next_block(68, additional_coinbase_value=10) tx = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue - 9) b68 = self.update_block(68, [tx]) - self.sync_blocks([b68], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b68], success=False, reject_reason='bad-cb-amount', reconnect=True) self.log.info("Accept a block claiming the correct subsidy in the coinbase transaction") self.move_tip(65) b69 = self.next_block(69, additional_coinbase_value=10) tx = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue - 10) self.update_block(69, [tx]) - self.sync_blocks([b69], True) + self.send_blocks([b69], True) self.save_spendable_output() # Test spending the outpoint of a non-existent transaction @@ -975,7 +961,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff)) tx.vout.append(CTxOut(1, b"")) b70 = self.update_block(70, [tx]) - self.sync_blocks([b70], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b70], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Test accepting an invalid block which has the same hash as a valid one (via merkle tree tricks) # @@ -1000,10 +986,10 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(b72.sha256, b71.sha256) self.move_tip(71) - self.sync_blocks([b71], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b71], success=False, reject_reason='bad-txns-duplicate', reconnect=True) self.move_tip(72) - self.sync_blocks([b72], True) + self.send_blocks([b72], True) self.save_spendable_output() # Test some invalid scripts and MAX_BLOCK_SIGOPS @@ -1038,7 +1024,7 @@ class FullBlockTest(BitcoinTestFramework): tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b73 = self.update_block(73, [tx]) assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS + 1) - self.sync_blocks([b73], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b73], success=False, reject_reason='bad-blk-sigops', reconnect=True) # b74/75 - if we push an invalid script element, all previous sigops are counted, # but sigops after the element are not counted. @@ -1062,7 +1048,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS + 4] = 0xff tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b74 = self.update_block(74, [tx]) - self.sync_blocks([b74], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b74], success=False, reject_reason='bad-blk-sigops', reconnect=True) self.move_tip(72) b75 = self.next_block(75) @@ -1075,7 +1061,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS + 3] = 0xff tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b75 = self.update_block(75, [tx]) - self.sync_blocks([b75], True) + self.send_blocks([b75], True) self.save_spendable_output() # Check that if we push an element filled with CHECKSIGs, they are not counted @@ -1086,7 +1072,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS - 1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs tx = self.create_and_sign_transaction(out[23], 1, CScript(a)) b76 = self.update_block(76, [tx]) - self.sync_blocks([b76], True) + self.send_blocks([b76], True) self.save_spendable_output() # Test transaction resurrection @@ -1111,40 +1097,40 @@ class FullBlockTest(BitcoinTestFramework): b77 = self.next_block(77) tx77 = self.create_and_sign_transaction(out[24], 10 * COIN) b77 = self.update_block(77, [tx77]) - self.sync_blocks([b77], True) + self.send_blocks([b77], True) self.save_spendable_output() b78 = self.next_block(78) tx78 = self.create_tx(tx77, 0, 9 * COIN) b78 = self.update_block(78, [tx78]) - self.sync_blocks([b78], True) + self.send_blocks([b78], True) b79 = self.next_block(79) tx79 = self.create_tx(tx78, 0, 8 * COIN) b79 = self.update_block(79, [tx79]) - self.sync_blocks([b79], True) + self.send_blocks([b79], True) # mempool should be empty assert_equal(len(self.nodes[0].getrawmempool()), 0) self.move_tip(77) b80 = self.next_block(80, spend=out[25]) - self.sync_blocks([b80], False, force_send=True) + self.send_blocks([b80], False, force_send=True) self.save_spendable_output() b81 = self.next_block(81, spend=out[26]) - self.sync_blocks([b81], False, force_send=True) # other chain is same length + self.send_blocks([b81], False, force_send=True) # other chain is same length self.save_spendable_output() b82 = self.next_block(82, spend=out[27]) - self.sync_blocks([b82], True) # now this chain is longer, triggers re-org + self.send_blocks([b82], True) # now this chain is longer, triggers re-org self.save_spendable_output() # now check that tx78 and tx79 have been put back into the peer's mempool mempool = self.nodes[0].getrawmempool() assert_equal(len(mempool), 2) - assert(tx78.hash in mempool) - assert(tx79.hash in mempool) + assert tx78.hash in mempool + assert tx79.hash in mempool # Test invalid opcodes in dead execution paths. # @@ -1161,7 +1147,7 @@ class FullBlockTest(BitcoinTestFramework): tx2.rehash() b83 = self.update_block(83, [tx1, tx2]) - self.sync_blocks([b83], True) + self.send_blocks([b83], True) self.save_spendable_output() # Reorg on/off blocks that have OP_RETURN in them (and try to spend them) @@ -1188,30 +1174,30 @@ class FullBlockTest(BitcoinTestFramework): tx5 = self.create_tx(tx1, 4, 0, CScript([OP_RETURN])) b84 = self.update_block(84, [tx1, tx2, tx3, tx4, tx5]) - self.sync_blocks([b84], True) + self.send_blocks([b84], True) self.save_spendable_output() self.move_tip(83) b85 = self.next_block(85, spend=out[29]) - self.sync_blocks([b85], False) # other chain is same length + self.send_blocks([b85], False) # other chain is same length b86 = self.next_block(86, spend=out[30]) - self.sync_blocks([b86], True) + self.send_blocks([b86], True) self.move_tip(84) b87 = self.next_block(87, spend=out[30]) - self.sync_blocks([b87], False) # other chain is same length + self.send_blocks([b87], False) # other chain is same length self.save_spendable_output() b88 = self.next_block(88, spend=out[31]) - self.sync_blocks([b88], True) + self.send_blocks([b88], True) self.save_spendable_output() # trying to spend the OP_RETURN output is rejected b89a = self.next_block("89a", spend=out[32]) tx = self.create_tx(tx1, 0, 0, CScript([OP_TRUE])) b89a = self.update_block("89a", [tx]) - self.sync_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) self.log.info("Test a re-org of one week's worth of blocks (1088 blocks)") @@ -1232,7 +1218,7 @@ class FullBlockTest(BitcoinTestFramework): self.save_spendable_output() spend = self.get_spendable_output() - self.sync_blocks(blocks, True, timeout=480) + self.send_blocks(blocks, True, timeout=480) chain1_tip = i # now create alt chain of same length @@ -1240,22 +1226,22 @@ class FullBlockTest(BitcoinTestFramework): blocks2 = [] for i in range(89, LARGE_REORG_SIZE + 89): blocks2.append(self.next_block("alt" + str(i), version=4)) - self.sync_blocks(blocks2, False, force_send=True) + self.send_blocks(blocks2, False, force_send=True) # extend alt chain to trigger re-org block = self.next_block("alt" + str(chain1_tip + 1), version=4) - self.sync_blocks([block], True, timeout=480) + self.send_blocks([block], True, timeout=480) # ... and re-org back to the first chain self.move_tip(chain1_tip) block = self.next_block(chain1_tip + 1, version=4) - self.sync_blocks([block], False, force_send=True) + self.send_blocks([block], False, force_send=True) block = self.next_block(chain1_tip + 2, version=4) - self.sync_blocks([block], True, timeout=480) + self.send_blocks([block], True, timeout=480) self.log.info("Reject a block with an invalid block header version") b_v1 = self.next_block('b_v1', version=1) - self.sync_blocks([b_v1], success=False, force_send=True, reject_reason='bad-version(0x00000001)') + self.send_blocks([b_v1], success=False, force_send=True, reject_reason='bad-version(0x00000001)', reconnect=True) self.move_tip(chain1_tip + 2) b_cb34 = self.next_block('b_cb34', version=4) @@ -1263,7 +1249,7 @@ class FullBlockTest(BitcoinTestFramework): b_cb34.vtx[0].rehash() b_cb34.hashMerkleRoot = b_cb34.calc_merkle_root() b_cb34.solve() - self.sync_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True) + self.send_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True) # Helper methods ################ @@ -1284,7 +1270,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin[0].scriptSig = CScript() return (sighash, err) = SignatureHash(spend_tx.vout[0].scriptPubKey, tx, 0, SIGHASH_ALL) - tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))]) + tx.vin[0].scriptSig = CScript([self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL]))]) def create_and_sign_transaction(self, spend_tx, value, script=CScript([OP_TRUE])): tx = self.create_tx(spend_tx, 0, value, script) @@ -1372,7 +1358,7 @@ class FullBlockTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.bootstrap_p2p(timeout=timeout) - def sync_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): + def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 96c5e49b0e..b16eafccca 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -15,7 +15,6 @@ from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, O from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, hex_str_to_bytes, ) @@ -114,7 +113,7 @@ class BIP65Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 4b3f6603a2..2b93e3c24d 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -25,6 +25,10 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('-dash=1\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: -dash=1, options in configuration file must be specified without leading -') + with open(inc_conf_file_path, 'w', encoding='utf8') as conf: + conf.write("wallet=foo\n") + self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on regtest network when in [regtest] section.') + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: nono, if you intended to specify a negated option, use nono=1 instead') @@ -41,13 +45,21 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided') + inc_conf_file2_path = os.path.join(self.nodes[0].datadir, 'include2.conf') + with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: + conf.write('includeconf={}\n'.format(inc_conf_file2_path)) + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('testnot.datadir=1\n[testnet]\n') + conf.write('testnot.datadir=1\n') + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('') # clear def run_test(self): self.stop_node(0) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index df79c4312c..887e9dafa3 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test activation of the first version bits soft fork. @@ -106,7 +106,7 @@ def send_generic_input_tx(node, coinbases, address): def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip68inputs) >= 16) + assert len(bip68inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip68inputs[i], address, amount=Decimal("49.98")) @@ -121,7 +121,7 @@ def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): def create_bip112txs(node, bip112inputs, varyOP_CSV, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip112inputs) >= 16) + assert len(bip112inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip112inputs[i], address, amount=Decimal("49.98")) @@ -168,7 +168,7 @@ class BIP68_112_113Test(BitcoinTestFramework): block.solve() return block - def sync_blocks(self, blocks, success=True): + def send_blocks(self, blocks, success=True): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" @@ -190,7 +190,7 @@ class BIP68_112_113Test(BitcoinTestFramework): self.log.info("Test that the csv softfork is DEFINED") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'defined') test_blocks = self.generate_blocks(61, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Advance from DEFINED to STARTED, height = 143") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'started') @@ -202,7 +202,7 @@ class BIP68_112_113Test(BitcoinTestFramework): test_blocks = self.generate_blocks(20, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, 536871169, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(24, 536936448, test_blocks) # 0x20010000 (signalling not) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Failed to advance past STARTED, height = 287") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'started') @@ -214,14 +214,14 @@ class BIP68_112_113Test(BitcoinTestFramework): test_blocks = self.generate_blocks(26, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, 536871169, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(10, 536936448, test_blocks) # 0x20010000 (signalling not) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Advanced from STARTED to LOCKED_IN, height = 431") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'locked_in') # Generate 140 more version 4 blocks test_blocks = self.generate_blocks(140, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # Inputs at height = 572 # @@ -264,7 +264,7 @@ class BIP68_112_113Test(BitcoinTestFramework): # 2 more version 4 blocks test_blocks = self.generate_blocks(2, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Not yet advanced to ACTIVE, height = 574 (will activate for block 576, not 575)") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'locked_in') @@ -318,7 +318,7 @@ class BIP68_112_113Test(BitcoinTestFramework): # try BIP 112 with seq=9 txs success_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v1)) success_txs.extend(all_rlt_txs(bip112txs_vary_OP_CSV_9_v1)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("Test version 2 txs") @@ -337,12 +337,12 @@ class BIP68_112_113Test(BitcoinTestFramework): # try BIP 112 with seq=9 txs success_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v2)) success_txs.extend(all_rlt_txs(bip112txs_vary_OP_CSV_9_v2)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # 1 more version 4 block to get us to height 575 so the fork should now be active for the next block test_blocks = self.generate_blocks(1, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'active') self.log.info("Post-Soft Fork Tests.") @@ -354,74 +354,74 @@ class BIP68_112_113Test(BitcoinTestFramework): bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block bip113signed2 = sign_transaction(self.nodes[0], bip113tx_v2) for bip113tx in [bip113signed1, bip113signed2]: - self.sync_blocks([self.create_test_block([bip113tx])], success=False) + self.send_blocks([self.create_test_block([bip113tx])], success=False) # BIP 113 tests should now pass if the locktime is < MTP bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block bip113signed1 = sign_transaction(self.nodes[0], bip113tx_v1) bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block bip113signed2 = sign_transaction(self.nodes[0], bip113tx_v2) for bip113tx in [bip113signed1, bip113signed2]: - self.sync_blocks([self.create_test_block([bip113tx])]) + self.send_blocks([self.create_test_block([bip113tx])]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # Next block height = 580 after 4 blocks of random version test_blocks = self.generate_blocks(4, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("BIP 68 tests") self.log.info("Test version 1 txs - all should still pass") success_txs = [] success_txs.extend(all_rlt_txs(bip68txs_v1)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("Test version 2 txs") # All txs with SEQUENCE_LOCKTIME_DISABLE_FLAG set pass bip68success_txs = [tx['tx'] for tx in bip68txs_v2 if tx['sdf']] - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # All txs without flag fail as we are at delta height = 8 < 10 and delta time = 8 * 600 < 10 * 512 bip68timetxs = [tx['tx'] for tx in bip68txs_v2 if not tx['sdf'] and tx['stf']] for tx in bip68timetxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) bip68heighttxs = [tx['tx'] for tx in bip68txs_v2 if not tx['sdf'] and not tx['stf']] for tx in bip68heighttxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Advance one block to 581 test_blocks = self.generate_blocks(1, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # Height txs should fail and time txs should now pass 9 * 600 > 10 * 512 bip68success_txs.extend(bip68timetxs) - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) for tx in bip68heighttxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Advance one block to 582 test_blocks = self.generate_blocks(1, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # All BIP 68 txs should pass bip68success_txs.extend(bip68heighttxs) - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("BIP 112 tests") self.log.info("Test version 1 txs") # -1 OP_CSV tx should fail - self.sync_blocks([self.create_test_block([bip112tx_special_v1])], success=False) + self.send_blocks([self.create_test_block([bip112tx_special_v1])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 1 txs should still pass success_txs = [tx['tx'] for tx in bip112txs_vary_OP_CSV_v1 if tx['sdf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if tx['sdf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is unset in argument to OP_CSV, version 1 txs should now fail @@ -430,18 +430,18 @@ class BIP68_112_113Test(BitcoinTestFramework): fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if not tx['sdf']] fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if not tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) self.log.info("Test version 2 txs") # -1 OP_CSV tx should fail - self.sync_blocks([self.create_test_block([bip112tx_special_v2])], success=False) + self.send_blocks([self.create_test_block([bip112tx_special_v2])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 2 txs should pass (all sequence locks are met) success_txs = [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if tx['sdf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v2 if tx['sdf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # SEQUENCE_LOCKTIME_DISABLE_FLAG is unset in argument to OP_CSV for all remaining txs ## @@ -450,23 +450,23 @@ class BIP68_112_113Test(BitcoinTestFramework): fail_txs = all_rlt_txs(bip112txs_vary_nSequence_9_v2) fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v2 if not tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in nSequence, tx should fail fail_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # If sequencelock types mismatch, tx should fail fail_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if not tx['sdf'] and tx['stf']] fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and tx['stf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Remaining txs should pass, just test masking works properly success_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if not tx['sdf'] and not tx['stf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and not tx['stf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # Additional test, of checking that comparison of two time types works properly @@ -476,7 +476,7 @@ class BIP68_112_113Test(BitcoinTestFramework): signtx = sign_transaction(self.nodes[0], tx) time_txs.append(signtx) - self.sync_blocks([self.create_test_block(time_txs)]) + self.send_blocks([self.create_test_block(time_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # TODO: Test empty stack fails diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 8b06cc7372..62062926a6 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. @@ -28,18 +28,12 @@ import errno import http.client import random -import sys import time from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, ToHex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, create_confirmed_utxos, hex_str_to_bytes -HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest] -try: - HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected) -except AttributeError: - pass class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): @@ -110,14 +104,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): try: self.nodes[node_index].submitblock(block) return True - except http.client.BadStatusLine as e: - # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error. - if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''": - self.log.debug("node %d submitblock raised exception: %s", node_index, e) - return False - else: - raise - except tuple(HTTP_DISCONNECT_ERRORS) as e: + except (http.client.CannotSendRequest, http.client.RemoteDisconnected) as e: self.log.debug("node %d submitblock raised exception: %s", node_index, e) return False except OSError as e: diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 8ff9498ef3..7480e5c5ba 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -14,7 +14,6 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, wait_until, ) @@ -103,7 +102,7 @@ class BIP66Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index b68e46adbc..a4b9f213a1 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" @@ -15,8 +15,6 @@ from test_framework.util import ( assert_greater_than_or_equal, connect_nodes, satoshi_round, - sync_blocks, - sync_mempools, ) # Construct 2 trivial P2SH's and the ScriptSigs that spend them @@ -65,7 +63,7 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee # the ScriptSig that will satisfy the ScriptPubKey. for inp in tx.vin: inp.scriptSig = SCRIPT_SIG[inp.prevout.n] - txid = from_node.sendrawtransaction(ToHex(tx), True) + txid = from_node.sendrawtransaction(hexstring=ToHex(tx), maxfeerate=0) unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) unconflist.append({"txid": txid, "vout": 1, "amount": amount}) @@ -95,7 +93,7 @@ def split_inputs(from_node, txins, txouts, initial_split=False): else: tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] completetx = ToHex(tx) - txid = from_node.sendrawtransaction(completetx, True) + txid = from_node.sendrawtransaction(hexstring=completetx, maxfeerate=0) txouts.append({"txid": txid, "vout": 0, "amount": half_change}) txouts.append({"txid": txid, "vout": 1, "amount": rem_change}) @@ -162,9 +160,9 @@ class EstimateFeeTest(BitcoinTestFramework): self.memutxo, Decimal("0.005"), min_fee, min_fee) tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee) / tx_kbytes) - sync_mempools(self.nodes[0:3], wait=.1) + self.sync_mempools(self.nodes[0:3], wait=.1) mined = mining_node.getblock(mining_node.generate(1)[0], True)["tx"] - sync_blocks(self.nodes[0:3], wait=.1) + self.sync_blocks(self.nodes[0:3], wait=.1) # update which txouts are confirmed newmem = [] for utx in self.memutxo: @@ -237,7 +235,7 @@ class EstimateFeeTest(BitcoinTestFramework): while len(self.nodes[1].getrawmempool()) > 0: self.nodes[1].generate(1) - sync_blocks(self.nodes[0:3], wait=.1) + self.sync_blocks(self.nodes[0:3], wait=.1) self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 9fb0d35a68..ba116e41f5 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -23,7 +23,7 @@ class FilelockTest(BitcoinTestFramework): self.log.info("Using datadir {}".format(datadir)) self.log.info("Check that we can't start a second bitcoind instance using the same datadir") - expected_msg = "Error: Cannot obtain a lock on data directory {}. Bitcoin Core is probably already running.".format(datadir) + expected_msg = "Error: Cannot obtain a lock on data directory {0}. {1} is probably already running.".format(datadir, self.config['environment']['PACKAGE_NAME']) self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index b98dd6d26c..13cf951550 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index eb76089d9c..a56c983ccc 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -18,7 +18,7 @@ from test_framework.blocktools import create_coinbase, create_block, create_tran from test_framework.messages import CTransaction from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str +from test_framework.util import assert_equal, assert_raises_rpc_error NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" @@ -27,7 +27,7 @@ def trueDummy(tx): newscript = [] for i in scriptSig: if (len(newscript) == 0): - assert(len(i) == 0) + assert len(i) == 0 newscript.append(b'\x51') else: newscript.append(i) @@ -64,17 +64,17 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] - txid1 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[0].serialize_with_witness()), True) + txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) - txid2 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[1].serialize_with_witness()), True) + txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) - txid3 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[2].serialize_with_witness()), True) + txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test1txs, False, True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) trueDummy(test2tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], False, True) @@ -83,19 +83,19 @@ class NULLDUMMYTest(BitcoinTestFramework): test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test5tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test5tx], True) self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") for i in test6txs: - self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) + self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test6txs, True, True) def block_submit(self, node, txs, witness=False, accept=False): @@ -108,7 +108,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index f042adb7aa..be323d355e 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. @@ -95,7 +95,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv4 connection through node node.addnode("15.61.23.23:1234", "onetry") cmd = proxies[0].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"15.61.23.23") @@ -109,7 +109,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv6 connection through node node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"1233:3432:2434:2343:3234:2345:6546:4534") @@ -123,7 +123,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing onion connection through node node.addnode("bitcoinostk4e4re.onion:8333", "onetry") cmd = proxies[2].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"bitcoinostk4e4re.onion") assert_equal(cmd.port, 8333) @@ -135,7 +135,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing DNS name connection through node node.addnode("node.noumenon:8333", "onetry") cmd = proxies[3].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"node.noumenon") assert_equal(cmd.port, 8333) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 3e1ba88f0a..e2b3b2d544 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -8,11 +8,20 @@ WARNING: This test uses 4GB of disk space. This test takes 30 mins or more (up to 2 hours) """ +import os +from test_framework.blocktools import create_coinbase +from test_framework.messages import CBlock, ToHex +from test_framework.script import CScript, OP_RETURN, OP_NOP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until - -import os +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, + wait_until, +) MIN_BLOCKS_TO_KEEP = 288 @@ -21,19 +30,60 @@ MIN_BLOCKS_TO_KEEP = 288 # compatible with pruning based on key creation time. TIMESTAMP_WINDOW = 2 * 60 * 60 +def mine_large_blocks(node, n): + # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN + # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase + # transaction but is consensus valid. + + # Set the nTime if this is the first time this function has been called. + # A static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + if "nTimes" not in mine_large_blocks.__dict__: + mine_large_blocks.nTime = 0 + + # Get the block parameters for the first block + big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) + best_block = node.getblock(node.getbestblockhash()) + height = int(best_block["height"]) + 1 + mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1 + previousblockhash = int(best_block["hash"], 16) + + for _ in range(n): + # Build the coinbase transaction (with large scriptPubKey) + coinbase_tx = create_coinbase(height) + coinbase_tx.vin[0].nSequence = 2 ** 32 - 1 + coinbase_tx.vout[0].scriptPubKey = big_script + coinbase_tx.rehash() + + # Build the block + block = CBlock() + block.nVersion = best_block["version"] + block.hashPrevBlock = previousblockhash + block.nTime = mine_large_blocks.nTime + block.nBits = int('207fffff', 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Submit to the node + node.submitblock(ToHex(block)) + + previousblockhash = block.sha256 + height += 1 + mine_large_blocks.nTime += 1 def calc_usage(blockdir): - return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) + return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timeout = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ @@ -55,10 +105,10 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[0], 4) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def setup_nodes(self): self.add_nodes(self.num_nodes, self.extra_args) @@ -69,23 +119,21 @@ class PruneTest(BitcoinTestFramework): def create_big_chain(self): # Start by creating some coinbases we can spend later self.nodes[1].generate(200) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MiB of data - for i in range(645): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 645) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def test_height_min(self): - if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): - raise AssertionError("blk00000.dat is missing, pruning too early") + assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) @@ -93,8 +141,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") + assert_greater_than(550, usage) def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks @@ -103,90 +150,66 @@ class PruneTest(BitcoinTestFramework): for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects - # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine - self.stop_node(0) - self.start_node(0, extra_args=self.full_node_default_args) + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[0], 2) # Mine 24 blocks in node 1 - for i in range(24): - if j == 0: - mine_large_block(self.nodes[1], self.utxo_cache_1) - else: - # Add node1's wallet transactions back to the mempool, to - # avoid the mined blocks from being too small. - self.nodes[1].resendwallettransactions() - self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + mine_large_blocks(self.nodes[1], 24) # Reorg back with 25 block chain from node 0 - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Create connections in the order so both nodes can see the reorg at the same time - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) - sync_blocks(self.nodes[0:3]) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + self.sync_blocks(self.nodes[0:3]) self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain - # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) - # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) - invalidheight = height-287 - badhash = self.nodes[1].getblockhash(invalidheight) - self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) - self.nodes[1].invalidateblock(badhash) + self.forkheight = height - 287 + self.forkhash = self.nodes[1].getblockhash(self.forkheight) + self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) - mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) while curhash != mainchainhash: self.nodes[1].invalidateblock(curhash) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) - assert(self.nodes[1].getblockcount() == invalidheight - 1) + assert self.nodes[1].getblockcount() == self.forkheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) - # Reboot node1 to clear those giant tx's from mempool - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) + # Disconnect node1 and generate the new chain + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[1], 2) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[2], 1) - sync_blocks(self.nodes[0:3], timeout=120) + connect_nodes(self.nodes[1], 2) + self.sync_blocks(self.nodes[0:3], timeout=120) self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) - - self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) - # Get node0's wallet transactions back in its mempool, to avoid the - # mined blocks from being too small. - self.nodes[0].resendwallettransactions() + self.log.info("Mine 220 more large blocks so we have requisite history") - for i in range(22): - # This can be slow, so do this in multiple RPC calls to avoid - # RPC timeouts. - self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects - sync_blocks(self.nodes[0:3], timeout=300) + mine_large_blocks(self.nodes[0], 220) usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") - - return invalidheight,badhash + assert_greater_than(550, usage) def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away @@ -219,17 +242,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert(self.nodes[0].getblockcount() == self.mainchainheight) - assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) + assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert(self.nodes[2].getbestblockhash() == goalbesthash) + assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned - assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode @@ -287,38 +310,30 @@ class PruneTest(BitcoinTestFramework): # height=100 too low to prune first block file so this is a no-op prune(100) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # Does nothing node.pruneblockchain(height(0)) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # height=500 should prune first file prune(500) - if has_block(0): - raise AssertionError("blk00000.dat is still there, should be pruned by now") - if not has_block(1): - raise AssertionError("blk00001.dat is missing when should still be there") + assert not has_block(0), "blk00000.dat is still there, should be pruned by now" + assert has_block(1), "blk00001.dat is missing when should still be there" # height=650 should prune second file prune(650) - if has_block(1): - raise AssertionError("blk00001.dat is still there, should be pruned by now") + assert not has_block(1), "blk00001.dat is still there, should be pruned by now" # height=1000 should not prune anything more, because tip-288 is in blk00002.dat. prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) - if not has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") + assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) node.generate(288) prune(1000) - if has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") - if has_block(3): - raise AssertionError("blk00003.dat is still there, should be pruned by now") + assert not has_block(2), "blk00002.dat is still there, should be pruned by now" + assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) @@ -338,22 +353,15 @@ class PruneTest(BitcoinTestFramework): self.log.info("Syncing node 5 to test wallet") connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] - sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) #stop and start to trigger rescan + self.sync_blocks(nds, wait=5, timeout=300) + self.stop_node(5) # stop and start to trigger rescan self.start_node(5, extra_args=["-prune=550"]) self.log.info("Success") def run_test(self): - self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - self.log.info("Mining a big blockchain of 995 blocks") - - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] + self.log.info("Warning! This test requires 4GB of disk space") + self.log.info("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -394,11 +402,11 @@ class PruneTest(BitcoinTestFramework): # +...+(1044) &.. $...$(1319) # Save some current chain state for later use - self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.log.info("Check that we can survive a 288 block reorg still") - (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + self.reorg_test() # (1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index ff7c2b23bf..ccba547a1c 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -9,12 +9,12 @@ from decimal import Decimal from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, satoshi_round +from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round MAX_REPLACEMENT_LIMIT = 100 def txToHex(tx): - return bytes_to_hex_str(tx.serialize()) + return tx.serialize().hex() def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): """Create a txout with a given amount and scriptPubKey @@ -46,7 +46,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): signed_tx = node.signrawtransactionwithwallet(txToHex(tx2)) - txid = node.sendrawtransaction(signed_tx['hex'], True) + txid = node.sendrawtransaction(signed_tx['hex'], 0) # If requested, ensure txouts are confirmed. if confirmed: @@ -56,7 +56,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): new_size = len(node.getrawmempool()) # Error out if we have something stuck in the mempool, as this # would likely be a bug. - assert(new_size < mempool_size) + assert new_size < mempool_size mempool_size = new_size return COutPoint(int(txid, 16), 0) @@ -136,7 +136,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) self.sync_all() @@ -147,9 +147,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # This will raise an exception due to transaction replacement being disabled - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Extra 0.1 BTC fee tx1b = CTransaction() @@ -157,14 +157,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(int(0.9 * COIN), CScript([b'b' * 35]))] tx1b_hex = txToHex(tx1b) # Replacement still disabled even with "enough fee" - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Works when enabled - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) mempool = self.nodes[0].getrawmempool() - assert (tx1a_txid not in mempool) - assert (tx1b_txid in mempool) + assert tx1a_txid not in mempool + assert tx1b_txid in mempool assert_equal(tx1b_hex, self.nodes[0].getrawtransaction(tx1b_txid)) @@ -188,7 +188,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vin = [CTxIn(prevout, nSequence=0)] tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] tx_hex = txToHex(tx) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) chain_txids.append(txid) prevout = COutPoint(int(txid, 16), 0) @@ -200,18 +200,18 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for doublespent_txid in chain_txids: - assert(doublespent_txid not in mempool) + assert doublespent_txid not in mempool def test_doublespend_tree(self): """Doublespend of a big tree of transactions""" @@ -236,8 +236,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vout = vout tx_hex = txToHex(tx) - assert(len(tx.serialize()) < 100000) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + assert len(tx.serialize()) < 100000 + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) yield tx _total_txs[0] += 1 @@ -261,20 +261,20 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 1 BTC fee is enough dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(initial_nValue - fee * n - 1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for tx in tree_txs: tx.rehash() - assert (tx.hash not in mempool) + assert tx.hash not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -289,7 +289,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) for tx in tree_txs: tx.rehash() @@ -303,7 +303,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - self.nodes[0].sendrawtransaction(tx1a_hex, True) + self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. @@ -313,7 +313,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) def test_spends_of_conflicting_outputs(self): """Replacements that spend conflicting tx outputs are rejected""" @@ -324,7 +324,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(utxo1, nSequence=0)] tx1a.vout = [CTxOut(int(1.1 * COIN), CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) tx1a_txid = int(tx1a_txid, 16) @@ -336,14 +336,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. tx1b = CTransaction() tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] tx1b.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1b_hex = txToHex(tx1b) - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) tx1b_txid = int(tx1b_txid, 16) tx2 = CTransaction() @@ -353,7 +353,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" @@ -364,7 +364,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1.vin = [CTxIn(confirmed_utxo)] tx1.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1_hex = txToHex(tx1) - self.nodes[0].sendrawtransaction(tx1_hex, True) + self.nodes[0].sendrawtransaction(tx1_hex, 0) tx2 = CTransaction() tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] @@ -372,7 +372,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_too_many_replacements(self): """Replacements that evict too many transactions are rejected""" @@ -394,7 +394,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): splitting_tx.vout = outputs splitting_tx_hex = txToHex(splitting_tx) - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, True) + txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) txid = int(txid, 16) # Now spend each of those outputs individually @@ -403,7 +403,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] tx_i.vout = [CTxOut(split_value - fee, CScript([b'a' * 35]))] tx_i_hex = txToHex(tx_i) - self.nodes[0].sendrawtransaction(tx_i_hex, True) + self.nodes[0].sendrawtransaction(tx_i_hex, 0) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have @@ -418,14 +418,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx_hex = txToHex(double_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass double_tx = CTransaction() double_tx.vin = inputs[0:-1] double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] double_tx_hex = txToHex(double_tx) - self.nodes[0].sendrawtransaction(double_tx_hex, True) + self.nodes[0].sendrawtransaction(double_tx_hex, 0) def test_opt_in(self): """Replacing should only work if orig tx opted in""" @@ -436,7 +436,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0xffffffff)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # This transaction isn't shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) @@ -448,7 +448,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -457,7 +457,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, True) + tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Still shouldn't be able to double-spend tx2b = CTransaction() @@ -466,7 +466,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now create a new transaction that spends from tx1a and tx2a # opt-in on one of the inputs @@ -481,7 +481,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3a.vout = [CTxOut(int(0.9*COIN), CScript([b'c'])), CTxOut(int(0.9*COIN), CScript([b'd']))] tx3a_hex = txToHex(tx3a) - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, True) + tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) @@ -496,10 +496,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3c.vout = [CTxOut(int(0.5 * COIN), CScript([b'f' * 35]))] tx3c_hex = txToHex(tx3c) - self.nodes[0].sendrawtransaction(tx3b_hex, True) + self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, True) + self.nodes[0].sendrawtransaction(tx3c_hex, 0) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -512,7 +512,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the actual fee per KB is much lower. tx1b = CTransaction() @@ -521,15 +521,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # Verify tx1b cannot replace tx1a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # Use prioritisetransaction to set tx1a's fee to 0. self.nodes[0].prioritisetransaction(txid=tx1a_txid, fee_delta=int(-0.1*COIN)) # Now tx1b should be able to replace tx1a - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - assert(tx1b_txid in self.nodes[0].getrawmempool()) + assert tx1b_txid in self.nodes[0].getrawmempool() # 2. Check that absolute fee checks use modified fee. tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -538,7 +538,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - self.nodes[0].sendrawtransaction(tx2a_hex, True) + self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Lower fee, but we'll prioritise it tx2b = CTransaction() @@ -548,15 +548,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now prioritise tx2b to have a higher modified fee self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1*COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) - assert(tx2b_txid in self.nodes[0].getrawmempool()) + assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): us0 = self.nodes[0].listunspent()[0] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 658a8cd75e..2d4dd96a1d 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -18,7 +18,13 @@ from test_framework.blocktools import witness_script, send_to_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, sha256, ToHex from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, + hex_str_to_bytes, + try_rpc, +) NODE_0 = 0 NODE_2 = 2 @@ -76,13 +82,13 @@ class SegWitTest(BitcoinTestFramework): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) block = node.generate(1) assert_equal(len(node.getblock(block[0])["tx"]), 2) - sync_blocks(self.nodes) + self.sync_blocks() def skip_mine(self, node, txid, sign, redeem_script=""): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) block = node.generate(1) assert_equal(len(node.getblock(block[0])["tx"]), 1) - sync_blocks(self.nodes) + self.sync_blocks() def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) @@ -93,17 +99,17 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -131,7 +137,7 @@ class SegWitTest(BitcoinTestFramework): p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) self.nodes[0].generate(1) # block 163 - sync_blocks(self.nodes) + self.sync_blocks() # Make sure all nodes recognize the transactions as theirs assert_equal(self.nodes[0].getbalance(), balance_presetup - 60 * 50 + 20 * Decimal("49.999") + 50) @@ -139,7 +145,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) self.nodes[0].generate(260) # block 423 - sync_blocks(self.nodes) + self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) # block 424 @@ -156,7 +162,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) blockhash = self.nodes[2].generate(1)[0] # block 432 (first block with new rules; 432 = 144 * 3) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) @@ -172,16 +178,16 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False)) - assert(self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False)) + assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) + assert self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False) for tx_id in segwit_tx_list: tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash)) - assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash)) - assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) == bytes_to_hex_str(tx.serialize_without_witness())) + assert self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) @@ -198,11 +204,11 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data - assert(tmpl['weightlimit'] == 4000000) - assert(tmpl['sigoplimit'] == 80000) - assert(tmpl['transactions'][0]['txid'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 8) + assert tmpl['sizelimit'] >= 3999577 # actual maximum size is lower due to minimum mandatory non-witness data + assert tmpl['weightlimit'] == 4000000 + assert tmpl['sigoplimit'] == 80000 + assert tmpl['transactions'][0]['txid'] == txid + assert tmpl['transactions'][0]['sigops'] == 8 self.nodes[0].generate(1) # Mine a block to clear the gbt cache @@ -214,8 +220,8 @@ class SegWitTest(BitcoinTestFramework): txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) hex_tx = self.nodes[0].gettransaction(txid)['hex'] tx = FromHex(CTransaction(), hex_tx) - assert(tx.wit.is_null()) # This should not be a segwit input - assert(txid1 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() # This should not be a segwit input + assert txid1 in self.nodes[0].getrawmempool() # Now create tx2, which will spend from txid1. tx = CTransaction() @@ -224,7 +230,7 @@ class SegWitTest(BitcoinTestFramework): tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx))['hex'] txid2 = self.nodes[0].sendrawtransaction(tx2_hex) tx = FromHex(CTransaction(), tx2_hex) - assert(not tx.wit.is_null()) + assert not tx.wit.is_null() # Now create tx3, which will spend from txid2 tx = CTransaction() @@ -232,15 +238,15 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(int(49.95 * COIN), CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) # Huge fee tx.calc_sha256() txid3 = self.nodes[0].sendrawtransaction(ToHex(tx)) - assert(tx.wit.is_null()) - assert(txid3 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() + assert txid3 in self.nodes[0].getrawmempool() # Check that getblocktemplate includes all transactions. template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) template_txids = [t['txid'] for t in template['transactions']] - assert(txid1 in template_txids) - assert(txid2 in template_txids) - assert(txid3 in template_txids) + assert txid1 in template_txids + assert txid2 in template_txids + assert txid3 in template_txids # Check that wtxid is properly reported in mempool entry assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) @@ -392,22 +398,22 @@ class SegWitTest(BitcoinTestFramework): v = self.nodes[0].getaddressinfo(i) if (v['isscript']): bare = hex_str_to_bytes(v['hex']) - importlist.append(bytes_to_hex_str(bare)) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(bare)]))) + importlist.append(bare.hex()) + importlist.append(CScript([OP_0, sha256(bare)]).hex()) else: pubkey = hex_str_to_bytes(v['pubkey']) p2pk = CScript([pubkey, OP_CHECKSIG]) p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) - importlist.append(bytes_to_hex_str(p2pk)) - importlist.append(bytes_to_hex_str(p2pkh)) - importlist.append(bytes_to_hex_str(CScript([OP_0, hash160(pubkey)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pk)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pkh)]))) + importlist.append(p2pk.hex()) + importlist.append(p2pkh.hex()) + importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) - importlist.append(bytes_to_hex_str(unsolvablep2pkh)) - importlist.append(bytes_to_hex_str(unsolvablep2wshp2pkh)) - importlist.append(bytes_to_hex_str(op1)) - importlist.append(bytes_to_hex_str(p2wshop1)) + importlist.append(unsolvablep2pkh.hex()) + importlist.append(unsolvablep2wshp2pkh.hex()) + importlist.append(op1.hex()) + importlist.append(p2wshop1.hex()) for i in importlist: # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC @@ -535,10 +541,10 @@ class SegWitTest(BitcoinTestFramework): for i in script_list: tx.vout.append(CTxOut(10000000, i)) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - txid = self.nodes[0].sendrawtransaction(signresults, True) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + txid = self.nodes[0].sendrawtransaction(signresults, 0) txs_mined[txid] = self.nodes[0].generate(1)[0] - sync_blocks(self.nodes) + self.sync_blocks() watchcount = 0 spendcount = 0 for i in self.nodes[0].listunspent(): @@ -587,10 +593,10 @@ class SegWitTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(int('0x' + i, 0), j))) tx.vout.append(CTxOut(0, CScript())) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - self.nodes[0].sendrawtransaction(signresults, True) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + self.nodes[0].sendrawtransaction(signresults, 0) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() if __name__ == '__main__': diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 1ced618d47..0713925141 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -74,8 +74,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): node.generatetoaddress(VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() - assert(not VB_PATTERN.match(node.getmininginfo()["warnings"])) - assert(not VB_PATTERN.match(node.getnetworkinfo()["warnings"])) + assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) + assert not VB_PATTERN.match(node.getnetworkinfo()["warnings"]) # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(node.p2p, VB_THRESHOLD, VB_UNKNOWN_VERSION) @@ -95,8 +95,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"]) + assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] + assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index aed439339f..7bb7044cc0 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" @@ -16,7 +16,7 @@ class TestBitcoinCli(BitcoinTestFramework): """Main test logic""" cli_response = self.nodes[0].cli("-version").send_cli() - assert("Bitcoin Core RPC client version" in cli_response) + assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") if self.is_wallet_compiled(): @@ -57,16 +57,14 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) - if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) - assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) - assert_equal(cli_get_info['testnet'], blockchain_info['chain'] == "test") + assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): + assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index 20889366e5..bb868d7115 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" @@ -30,14 +30,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -47,14 +47,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -64,8 +64,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is None) #now the connection must be closed after the response + assert b'"error":null' in out1 + assert conn.sock is None #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -76,7 +76,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) + assert b'"error":null' in out1 #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urllib.parse.urlparse(self.nodes[2].url) @@ -87,8 +87,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default + assert b'"error":null' in out1 + assert conn.sock is not None #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index f33040242c..a036dfc790 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" @@ -152,7 +152,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) chain_height, = unpack("i", output.read(4)) - response_hash = binascii.hexlify(output.read(32)[::-1]).decode('ascii') + response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine assert_equal(chain_height, 102) # chain height must be 102 @@ -252,7 +252,7 @@ class RESTTest (BitcoinTestFramework): resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) - blockhash = binascii.hexlify(resp_bytes[::-1]).decode('utf-8') + blockhash = resp_bytes[::-1].hex() assert_equal(blockhash, bb_hash) # Check invalid blockhashbyheight requests diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b6955d4492..49ae0fb1a9 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,12 +1,22 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than_or_equal +def expect_http_status(expected_http_status, expected_rpc_code, + fcn, *args): + try: + fcn(*args) + raise AssertionError("Expected RPC error %d, got none" % expected_rpc_code) + except JSONRPCException as exc: + assert_equal(exc.error["code"], expected_rpc_code) + assert_equal(exc.http_status, expected_http_status) + class RPCInterfaceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,9 +58,16 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_equal(result_by_id[3]['error'], None) assert result_by_id[3]['result'] is not None + def test_http_status_codes(self): + self.log.info("Testing HTTP status codes for JSON-RPC requests...") + + expect_http_status(404, -32601, self.nodes[0].invalidmethod) + expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + def run_test(self): self.test_getrpcinfo() self.test_batch_request() + self.test_http_status_codes() if __name__ == '__main__': diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 94fea37090..8e58c85c15 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" @@ -10,7 +10,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction from test_framework.util import ( assert_equal, - bytes_to_hex_str, hash256, ) from io import BytesIO @@ -94,17 +93,17 @@ class ZMQTest (BitcoinTestFramework): tx = CTransaction() tx.deserialize(BytesIO(hex)) tx.calc_sha256() - assert_equal(tx.hash, bytes_to_hex_str(txid)) + assert_equal(tx.hash, txid.hex()) # Should receive the generated block hash. - hash = bytes_to_hex_str(self.hashblock.receive()) + hash = self.hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. - assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) # Should receive the generated raw block. block = self.rawblock.receive() - assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) + assert_equal(genhashes[x], hash256(block[:80]).hex()) if self.is_wallet_compiled(): self.log.info("Wait for tx from second node") @@ -113,11 +112,11 @@ class ZMQTest (BitcoinTestFramework): # Should receive the broadcasted txid. txid = self.hashtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(txid)) + assert_equal(payment_txid, txid.hex()) # Should receive the broadcasted raw transaction. hex = self.rawtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) + assert_equal(payment_txid, hash256(hex).hex()) self.log.info("Test the getzmqnotifications RPC") diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index efe720b9b1..2bb5d8ab7d 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" @@ -27,7 +27,6 @@ from test_framework.script import ( from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str, hex_str_to_bytes, ) @@ -69,7 +68,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): inputs=[{'txid': coin['txid'], 'vout': coin['vout']}], outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], ))['hex'] - txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True) + txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, maxfeerate=0) node.generate(1) self.mempool_size = 0 self.check_mempool_result( @@ -101,10 +100,10 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) - node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True) + node.sendrawtransaction(hexstring=raw_tx_final, maxfeerate=0) self.mempool_size += 1 self.log.info('A transaction in the mempool') @@ -119,7 +118,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(fee * COIN) # Double the fee tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER + 1 # Now, opt out of RBF - raw_tx_0 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] + raw_tx_0 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( @@ -129,15 +128,15 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that conflicts with an unconfirmed tx') # Send the transaction that replaces the mempool transaction and opts out of replaceability - node.sendrawtransaction(hexstring=bytes_to_hex_str(tx.serialize()), allowhighfees=True) + node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) # take original raw_tx_0 tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '18: txn-mempool-conflict'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) self.log.info('A transaction with missing inputs, that never existed') @@ -146,14 +145,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'missing-inputs'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with missing inputs, that existed once in the past') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vin[0].prevout.n = 1 # Set vout to 1, to spend the other outpoint (49 coins) of the in-chain-tx we want to double spend - raw_tx_1 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] - txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, allowhighfees=True) + raw_tx_1 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] + txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, maxfeerate=0) # Now spend both to "clearly hide" the outputs, ie. remove the coins from the utxo set by spending them raw_tx_spend_both = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[ @@ -162,7 +161,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ], outputs=[{node.getnewaddress(): 0.1}] ))['hex'] - txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, allowhighfees=True) + txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, maxfeerate=0) node.generate(1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again @@ -184,17 +183,17 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Reference tx should be valid on itself self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with no outputs') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout = [] # Skip re-signing the transaction for context independent checks from now on - # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex']))) + # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-empty'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A really large transaction') @@ -202,7 +201,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with negative output value') @@ -210,7 +209,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue *= -1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-negative'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large output value') @@ -218,7 +217,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large sum of output values') @@ -227,7 +226,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-txouttotal-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with duplicate inputs') @@ -235,7 +234,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-inputs-duplicate'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A coinbase transaction') @@ -244,7 +243,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_coinbase_spent))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: coinbase'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('Some nonstandard transactions') @@ -252,19 +251,19 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nVersion = 3 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: version'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_0]) # Some non-standard script self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptpubkey'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptsig-not-pushonly'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) @@ -272,21 +271,21 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: tx-size'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0] = output_p2sh_burn tx.vout[0].nValue -= 1 # Make output smaller, such that it is dust for our policy self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: dust'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'\xff']) tx.vout = [tx.vout[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: multi-op-return'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A timelocked transaction') @@ -295,7 +294,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nLockTime = node.getblockcount() + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction that is locked by BIP68 sequence logic') @@ -304,8 +303,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Can skip re-signing the tx because of early rejection self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-BIP68-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index c0918893cd..351b27e94a 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" @@ -47,9 +47,9 @@ class MempoolLimitTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) self.log.info('The tx should be evicted by now') - assert(txid not in self.nodes[0].getrawmempool()) + assert txid not in self.nodes[0].getrawmempool() txdata = self.nodes[0].gettransaction(txid) - assert(txdata['confirmations'] == 0) #confirmation should still be 0 + assert txdata['confirmations'] == 0 #confirmation should still be 0 self.log.info('Check that mempoolminfee is larger than minrelytxfee') assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 9336547a6b..c7d241503a 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" @@ -8,7 +8,11 @@ from decimal import Decimal from test_framework.messages import COIN from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + satoshi_round, +) MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 @@ -33,7 +37,7 @@ class MempoolPackagesTest(BitcoinTestFramework): signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx['hex']) fulltx = node.getrawtransaction(txid, 1) - assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output + assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output return (txid, send_value) def run_test(self): @@ -58,9 +62,9 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(mempool), MAX_ANCESTORS) descendant_count = 1 descendant_fees = 0 - descendant_size = 0 + descendant_vsize = 0 - ancestor_size = sum([mempool[tx]['size'] for tx in mempool]) + ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool]) ancestor_count = MAX_ANCESTORS ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool]) @@ -79,15 +83,15 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_size += mempool[x]['size'] - assert_equal(mempool[x]['descendantsize'], descendant_size) + descendant_vsize += mempool[x]['vsize'] + assert_equal(mempool[x]['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct assert_equal(mempool[x]['ancestorcount'], ancestor_count) assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_size) - ancestor_size -= mempool[x]['size'] + assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) + ancestor_vsize -= mempool[x]['vsize'] ancestor_fees -= mempool[x]['fee'] ancestor_count -= 1 @@ -125,13 +129,13 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(v_ancestors), len(chain)-1) for x in v_ancestors.keys(): assert_equal(mempool[x], v_ancestors[x]) - assert(chain[-1] not in v_ancestors.keys()) + assert chain[-1] not in v_ancestors.keys() v_descendants = self.nodes[0].getmempooldescendants(chain[0], True) assert_equal(len(v_descendants), len(chain)-1) for x in v_descendants.keys(): assert_equal(mempool[x], v_descendants[x]) - assert(chain[0] not in v_descendants.keys()) + assert chain[0] not in v_descendants.keys() # Check that ancestor modified fees includes fee deltas from # prioritisetransaction @@ -163,7 +167,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) # Prioritise a transaction that has been mined, then add it back to the # mempool by using invalidateblock. @@ -228,7 +232,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Test reorg handling # First, the basics: self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash()) self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash()) @@ -283,12 +287,12 @@ class MempoolPackagesTest(BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction(inputs, outputs) signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) txid = self.nodes[0].sendrawtransaction(signedtx['hex']) - sync_mempools(self.nodes) + self.sync_mempools() # Now try to disconnect the tip on each node... self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - sync_blocks(self.nodes) + self.sync_blocks() if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index d74d4eaaf1..bb0169ee52 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -37,7 +37,6 @@ Test is as follows: """ from decimal import Decimal import os -import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, wait_until @@ -83,9 +82,10 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(1, extra_args=["-persistmempool=0"]) self.start_node(0) self.start_node(2) - # Give bitcoind a second to reload the mempool - wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5, timeout=1) - wait_until(lambda: len(self.nodes[2].getrawmempool()) == 5, timeout=1) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) + wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) + assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -100,14 +100,14 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) - # Give bitcoind a second to reload the mempool - time.sleep(1) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) - wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) + assert_equal(len(self.nodes[0].getrawmempool()), 5) mempooldat0 = os.path.join(self.nodes[0].datadir, 'regtest', 'mempool.dat') mempooldat1 = os.path.join(self.nodes[1].datadir, 'regtest', 'mempool.dat') @@ -120,7 +120,8 @@ class MempoolPersistTest(BitcoinTestFramework): os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) - wait_until(lambda: len(self.nodes[1].getrawmempool()) == 5) + wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) + assert_equal(len(self.nodes[1].getrawmempool()), 5) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 845beb551e..187c9026f6 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" @@ -45,7 +45,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 # Use invalidateblock to re-org back for node in self.nodes: @@ -55,7 +55,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] == 0) + assert tx["confirmations"] == 0 # Generate another block, they should all get mined self.nodes[0].generate(1) @@ -63,7 +63,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 if __name__ == '__main__': diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index d0bef48cb6..788aabc192 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -27,7 +27,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str as b2x, connect_nodes_bi, ) from test_framework.script import CScriptNum @@ -36,7 +35,7 @@ from test_framework.script import CScriptNum def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() - rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']}) + rsp = node.getblocktemplate(template_request={'data': block.serialize().hex(), 'mode': 'proposal', 'rules': ['segwit']}) assert_equal(rsp, expect) @@ -64,8 +63,8 @@ class MiningTest(BitcoinTestFramework): def assert_submitblock(block, result_str_1, result_str_2=None): block.solve() result_str_2 = result_str_2 or 'duplicate-invalid' - assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize()))) - assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize()))) + assert_equal(result_str_1, node.submitblock(hexdata=block.serialize().hex())) + assert_equal(result_str_2, node.submitblock(hexdata=block.serialize().hex())) self.log.info('getmininginfo') mining_info = node.getmininginfo() @@ -112,7 +111,7 @@ class MiningTest(BitcoinTestFramework): assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, block.serialize()[:-15].hex()) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -121,10 +120,10 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': block.serialize()[:-1].hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -153,7 +152,7 @@ class MiningTest(BitcoinTestFramework): bad_block_sn = bytearray(block.serialize()) assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1) bad_block_sn[BLOCK_HEADER_SIZE] += 1 - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': bad_block_sn.hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) @@ -184,7 +183,7 @@ class MiningTest(BitcoinTestFramework): self.log.info('submitheader tests') assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE)) assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2))) - assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=b2x(super(CBlock, bad_block).serialize()))) + assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=super(CBlock, bad_block).serialize().hex())) block.nTime += 1 block.solve() @@ -193,23 +192,23 @@ class MiningTest(BitcoinTestFramework): return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status} assert chain_tip(block.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(block.serialize())) + node.submitheader(hexdata=block.serialize().hex()) assert chain_tip(block.hash) in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) # Noop + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) # Noop assert chain_tip(block.hash) in node.getchaintips() bad_block_root = copy.deepcopy(block) bad_block_root.hashMerkleRoot += 2 bad_block_root.solve() assert chain_tip(bad_block_root.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() # Should still reject invalid blocks, even if we have the header: - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') assert chain_tip(bad_block_root.hash) in node.getchaintips() # We know the header for this invalid block, so should just return early without error: - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() bad_block_lock = copy.deepcopy(block) @@ -217,19 +216,19 @@ class MiningTest(BitcoinTestFramework): bad_block_lock.vtx[0].rehash() bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root() bad_block_lock.solve() - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal') - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'bad-txns-nonfinal') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'duplicate-invalid') # Build a "good" block on top of the submitted bad block bad_block2 = copy.deepcopy(block) bad_block2.hashPrevBlock = bad_block_lock.sha256 bad_block2.solve() - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) # Should reject invalid header right away bad_block_time = copy.deepcopy(block) bad_block_time.nTime = 1 bad_block_time.solve() - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) # Should ask for the block from a p2p node, if they announce the header as well: node.add_p2p_connection(P2PDataStore()) @@ -240,11 +239,11 @@ class MiningTest(BitcoinTestFramework): # Building a few blocks should give the same results node.generatetoaddress(10, node.get_deterministic_priv_key().address) - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) - assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate') # valid + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) + assert_equal(node.submitblock(hexdata=block.serialize().hex()), 'duplicate') # valid if __name__ == '__main__': diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 72cde8e811..445ec124ce 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" @@ -38,27 +38,27 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): longpollid = template['longpollid'] # longpollid should not change between successive invocations if nothing else happens template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template2['longpollid'] == longpollid) + assert template2['longpollid'] == longpollid # Test 1: test that the longpolling wait if we do nothing thr = LongpollThread(self.nodes[0]) thr.start() # check that thread still lives thr.join(5) # wait 5 seconds or until thread exits - assert(thr.is_alive()) + assert thr.is_alive() # Test 2: test that longpoll will terminate if another node generates a block self.nodes[1].generate(1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 3: test that longpoll will terminate if we generate a block ourselves thr = LongpollThread(self.nodes[0]) thr.start() self.nodes[0].generate(1) # generate a block on another node thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll thr = LongpollThread(self.nodes[0]) @@ -69,7 +69,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) - assert(not thr.is_alive()) + assert not thr.is_alive() if __name__ == '__main__': GetBlockTemplateLPTest().main() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index ca4b621a78..b0a069be81 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -63,9 +63,9 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes = [0, 0, 0] for i in range(3): for j in txids[i]: - assert(j in mempool) - sizes[i] += mempool[j]['size'] - assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count + assert j in mempool + sizes[i] += mempool[j]['vsize'] + assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined @@ -75,8 +75,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that prioritised transaction was mined") - assert(txids[0][0] not in mempool) - assert(txids[0][1] in mempool) + assert txids[0][0] not in mempool + assert txids[0][1] in mempool high_fee_tx = None for x in txids[2]: @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx is not None) + assert high_fee_tx is not None # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). @@ -95,7 +95,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Check to make sure our high fee rate tx is back in the mempool mempool = self.nodes[0].getrawmempool() - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool # Now verify the modified-high feerate transaction isn't mined before # the other high fee transactions. Keep mining until our mempool has @@ -107,14 +107,14 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # transactions should have been. mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool for x in txids[2]: if (x != high_fee_tx): - assert(x not in mempool) + assert x not in mempool # Create a free transaction. Should be rejected. utxo_list = self.nodes[0].listunspent() - assert(len(utxo_list) > 0) + assert len(utxo_list) > 0 utxo = utxo_list[0] inputs = [] @@ -127,7 +127,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # This will raise an exception due to min relay fee not being met assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) - assert(tx_id not in self.nodes[0].getrawmempool()) + assert tx_id not in self.nodes[0].getrawmempool() # This is a less than 1000-byte transaction, so just set the fee # to be the minimum for a 1000-byte transaction and check that it is @@ -136,7 +136,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) - assert(tx_id in self.nodes[0].getrawmempool()) + assert tx_id in self.nodes[0].getrawmempool() # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. @@ -147,7 +147,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].setmocktime(mock_time+10) new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template != new_template) + assert template != new_template if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index f0dee59b5c..3ca6bec254 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -7,19 +7,18 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ -from decimal import Decimal import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_bip9_status, satoshi_round, sync_blocks, wait_until +from test_framework.util import assert_equal, get_bip9_status, wait_until # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self): + def __init__(self, cmpct_version): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -27,6 +26,7 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() + self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -94,11 +94,7 @@ class TestP2PConn(P2PInterface): class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] + self.num_nodes = 1 self.utxos = [] def skip_test_if_missing_module(self): @@ -117,11 +113,10 @@ class CompactBlocksTest(BitcoinTestFramework): # Create 10 more anyone-can-spend utxo's for testing. def make_utxos(self): - # Doesn't matter which node we use, just use node0. block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) - self.nodes[0].generate(100) + self.segwit_node.send_and_ping(msg_block(block)) + assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 + self.nodes[0].generatetoaddress(100, self.nodes[0].getnewaddress(address_type="bech32")) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -135,10 +130,10 @@ class CompactBlocksTest(BitcoinTestFramework): block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() block2.solve() - self.test_node.send_and_ping(msg_block(block2)) + self.segwit_node.send_and_ping(msg_block(block2)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return + # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. @@ -149,7 +144,10 @@ class CompactBlocksTest(BitcoinTestFramework): # are made with compact blocks. # If old_node is passed in, request compact blocks with version=preferred-1 # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + def test_sendcmpct(self, test_node, old_node=None): + preferred_version = test_node.cmpct_version + node = self.nodes[0] + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) @@ -167,7 +165,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.clear_block_announcement() block_hash = int(node.generate(1)[0], 16) peer.wait_for_block_announcement(block_hash, timeout=30) - assert(peer.block_announced) + assert peer.block_announced with mininode_lock: assert predicate(peer), ( @@ -251,23 +249,18 @@ class CompactBlocksTest(BitcoinTestFramework): # This index will be too high prefilled_txn = PrefilledTransaction(1, block.vtx[0]) cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): + def test_compactblock_construction(self, test_node, use_witness_address=True): + version = test_node.cmpct_version + node = self.nodes[0] # Generate a bunch of transactions. node.generate(101) num_transactions = 25 address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.getnewaddress(address_type='bech32') - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) - node.generate(1) segwit_tx_generated = False for i in range(num_transactions): @@ -285,7 +278,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.wait_for_block_announcement(tip) # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) + self.request_cb_announcements(test_node) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -303,7 +296,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -319,7 +312,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -330,7 +323,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(header_and_shortids.header.sha256, block_hash) # Make sure the prefilled_txn appears to have included the coinbase - assert(len(header_and_shortids.prefilled_txn) >= 1) + assert len(header_and_shortids.prefilled_txn) >= 1 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) # Check that all prefilled_txn entries match what's in the block. @@ -345,7 +338,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) else: # Shouldn't have received a witness - assert(entry.tx.wit.is_null()) + assert entry.tx.wit.is_null() # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -375,7 +368,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Post-segwit: upgraded nodes would only make this request of cb-version-2, # NODE_WITNESS peers. Unupgraded nodes would still make this request of # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): + def test_compactblock_requests(self, test_node, segwit=True): + version = test_node.cmpct_version + node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: @@ -407,7 +402,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -440,14 +435,16 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we only receive getblocktxn requests for transactions that the # node needs, and that responding to them causes the block to be # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): + def test_getblocktxn_requests(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert("getblocktxn" in peer.last_message) + assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -487,7 +484,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in node.getrawmempool()) + assert block.vtx[1].hash in node.getrawmempool() # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. @@ -508,7 +505,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Clear out last request. with mininode_lock: @@ -519,13 +516,13 @@ class CompactBlocksTest(BitcoinTestFramework): test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert("getblocktxn" not in test_node.last_message) + assert "getblocktxn" not in test_node.last_message # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() + def test_incorrect_blocktxn_response(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -537,7 +534,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:6]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Send compact block comp_block = HeaderAndShortIDs() @@ -545,7 +542,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -569,7 +566,7 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG) + assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block @@ -579,7 +576,9 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) - def test_getblocktxn_handler(self, node, test_node, version): + def test_getblocktxn_handler(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 @@ -606,7 +605,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: # Witnesses should have been stripped - assert(tx.wit.is_null()) + assert tx.wit.is_null() else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) @@ -626,7 +625,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message - def test_compactblocks_not_at_tip(self, node, test_node): + def test_compactblocks_not_at_tip(self, test_node): + node = self.nodes[0] # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] @@ -669,7 +669,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(x["status"], "headers-only") found = True break - assert(found) + assert found # Requesting this block via getblocktxn should silently fail # (to avoid fingerprinting attacks). @@ -681,11 +681,8 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert "blocktxn" not in test_node.last_message - def activate_segwit(self, node): - node.generate(144 * 3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') - - def test_end_to_end_block_relay(self, node, listeners): + def test_end_to_end_block_relay(self, listeners): + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -706,8 +703,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert(len(self.utxos)) + def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + node = self.nodes[0] + assert len(self.utxos) utxo = self.utxos[0] block = self.build_block_with_transactions(node, utxo, 5) @@ -728,22 +726,24 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg) # Check that the tip didn't advance - assert(int(node.getbestblockhash(), 16) is not block.sha256) + assert int(node.getbestblockhash(), 16) is not block.sha256 test_node.sync_with_ping() # Helper for enabling cb announcements # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): + def request_cb_announcements(self, peer): + node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) msg = msg_sendcmpct() - msg.version = version + msg.version = peer.cmpct_version msg.announce = True peer.send_and_ping(msg) - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert(len(self.utxos)) + def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + node = self.nodes[0] + assert len(self.utxos) def announce_cmpct_block(node, peer): utxo = self.utxos.pop(0) @@ -764,7 +764,7 @@ class CompactBlocksTest(BitcoinTestFramework): delivery_peer.sync_with_ping() mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.sha256) @@ -783,7 +783,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.use_witness = True delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert(int(node.getbestblockhash(), 16) != block.sha256) + assert int(node.getbestblockhash(), 16) != block.sha256 msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 @@ -793,126 +793,55 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) # We will need UTXOs to construct transactions in later tests. self.make_utxos() - self.log.info("Running tests, pre-segwit activation:") + assert_equal(get_bip9_status(self.nodes[0], "segwit")["status"], 'active') self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) + self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) - - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + self.test_compactblock_construction(self.old_node) + self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) + self.test_compactblock_requests(self.segwit_node) self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.segwit_node) self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + self.test_getblocktxn_handler(self.segwit_node) + self.test_getblocktxn_handler(self.old_node) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.segwit_node) + self.test_compactblocks_not_at_tip(self.old_node) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.segwit_node) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.segwit_node, self.additional_segwit_node) # Test that if we submitblock to node1, we'll get a compact block # announcement to all peers. # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + self.request_cb_announcements(self.old_node) + self.request_cb_announcements(self.segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + self.test_invalid_tx_in_compactblock(self.segwit_node) + self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index d589519e45..7f901b1886 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" @@ -10,7 +10,7 @@ import time from test_framework.messages import msg_feefilter from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import sync_blocks, sync_mempools + def hashToHex(hash): return format(hash, '064x') @@ -50,14 +50,14 @@ class FeeFilterTest(BitcoinTestFramework): node0 = self.nodes[0] # Get out of IBD node1.generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[0].add_p2p_connection(TestP2PConn()) # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte @@ -65,13 +65,13 @@ class FeeFilterTest(BitcoinTestFramework): # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - sync_mempools(self.nodes) # must be sure node 0 has received all txs + self.sync_mempools() # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we # we can sync the test on receipt (if node1's txs were relayed, they'd @@ -82,13 +82,13 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() if __name__ == '__main__': diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 700fdf6e04..481d697e63 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" +import asyncio import os import struct from test_framework import messages -from test_framework.mininode import P2PDataStore +from test_framework.mininode import P2PDataStore, NetworkThread from test_framework.test_framework import BitcoinTestFramework @@ -143,8 +144,15 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_magic_bytes(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes - conn.magic_bytes = b'\x00\x11\x22\x32' + + def swap_magic_bytes(): + conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes + conn.magic_bytes = b'\x00\x11\x22\x32' + + # Call .result() to block until the atomic swap is complete, otherwise + # we might run into races later on + asyncio.run_coroutine_threadsafe(asyncio.coroutine(swap_magic_bytes)(), NetworkThread.network_event_loop).result() + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): conn.send_message(messages.msg_ping(nonce=0xff)) conn.wait_for_disconnect(timeout=1) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 336d34a81d..06049db54c 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. @@ -117,9 +117,9 @@ class P2PLeakTest(BitcoinTestFramework): wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) # Make sure no unexpected messages came in - assert(no_version_bannode.unexpected_msg == False) - assert(no_version_idlenode.unexpected_msg == False) - assert(no_verack_idlenode.unexpected_msg == False) + assert no_version_bannode.unexpected_msg == False + assert no_version_idlenode.unexpected_msg == False + assert no_verack_idlenode.unexpected_msg == False if __name__ == '__main__': diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 359880506e..573d5f5a5f 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_NETWORK_LIMITED. @@ -11,7 +11,13 @@ and that it responds to getdata requests for blocks correctly: from test_framework.messages import CInv, msg_getdata, msg_verack, NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, disconnect_nodes, connect_nodes_bi, sync_blocks, wait_until +from test_framework.util import ( + assert_equal, + disconnect_nodes, + connect_nodes_bi, + wait_until, +) + class P2PIgnoreInv(P2PInterface): firstAddrnServices = 0 @@ -60,7 +66,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") connect_nodes_bi(self.nodes, 0, 1) blocks = self.nodes[1].generatetoaddress(292, self.nodes[1].get_deterministic_priv_key().address) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) self.log.info("Make sure we can max retrieve block at tip-288.") node.send_getdata_for_block(blocks[1]) # last block in valid range @@ -86,7 +92,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): # because node 2 is in IBD and node 0 is a NODE_NETWORK_LIMITED peer, sync must not be possible connect_nodes_bi(self.nodes, 0, 2) try: - sync_blocks([self.nodes[0], self.nodes[2]], timeout=5) + self.sync_blocks([self.nodes[0], self.nodes[2]], timeout=5) except: pass # node2 must remain at height 0 @@ -96,7 +102,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 1, 2) # sync must be possible - sync_blocks(self.nodes) + self.sync_blocks() # disconnect all peers self.disconnect_all() @@ -108,7 +114,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) # sync must be possible, node 1 is no longer in IBD and should therefore connect to node 0 (NODE_NETWORK_LIMITED) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) if __name__ == '__main__': NodeNetworkLimitedTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8f8e89cf15..000c30646a 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" -from binascii import hexlify import math import random import struct import time from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER -from test_framework.key import CECKey, CPubKey +from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, CBlock, @@ -37,6 +36,7 @@ from test_framework.messages import ( ser_vector, sha256, uint256_from_str, + FromHex, ) from test_framework.mininode import ( P2PInterface, @@ -74,13 +74,11 @@ from test_framework.script import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, connect_nodes, disconnect_nodes, get_bip9_status, hex_str_to_bytes, - sync_blocks, - sync_mempools, + assert_raises_rpc_error, ) # The versionbit bit used to signal activation of SegWit @@ -104,7 +102,7 @@ def get_p2pkh_script(pubkeyhash): def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): """Add signature for a P2PK witness program.""" tx_hash = SegwitVersion1SignatureHash(script, tx_to, in_idx, hashtype, value) - signature = key.sign(tx_hash) + chr(hashtype).encode('latin-1') + signature = key.sign_ecdsa(tx_hash) + chr(hashtype).encode('latin-1') tx_to.wit.vtxinwit[in_idx].scriptWitness.stack = [signature, script] tx_to.rehash() @@ -273,6 +271,7 @@ class SegWitTest(BitcoinTestFramework): self.test_non_standard_witness() self.test_upgrade_after_activation() self.test_witness_sigops() + self.test_superfluous_witness() # Individual tests @@ -285,7 +284,7 @@ class SegWitTest(BitcoinTestFramework): func(self, *args, **kwargs) # Each subtest should leave some utxos for the next subtest assert self.utxo - sync_blocks(self.nodes) + self.sync_blocks() # Assert segwit status is as expected at end of subtest assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], self.segwit_status) @@ -317,7 +316,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_message(msg_witness_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed - assert(tx.hash in self.nodes[0].getrawmempool()) + assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) @@ -335,7 +334,7 @@ class SegWitTest(BitcoinTestFramework): # Verify the hash with witness differs from the txid # (otherwise our testing framework must be broken!) tx.rehash() - assert(tx.sha256 != tx.calc_sha256(with_witness=True)) + assert tx.sha256 != tx.calc_sha256(with_witness=True) # Construct a segwit-signaling block that includes the transaction. block = self.build_next_block(version=(VB_TOP_BITS | (1 << VB_WITNESS_BIT))) @@ -371,20 +370,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) block2 = self.build_next_block(version=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block2, True) block3 = self.build_next_block(version=(VB_TOP_BITS | (1 << 15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -413,8 +412,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, []) # This gives us a witness commitment. - assert(len(block.vtx[0].wit.vtxinwit) == 1) - assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) + assert len(block.vtx[0].wit.vtxinwit) == 1 + assert len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1 test_witness_block(self.nodes[0], self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) @@ -448,7 +447,7 @@ class SegWitTest(BitcoinTestFramework): msg.headers = [CBlockHeader(block4)] self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) - assert(block4.sha256 not in self.old_node.getdataset) + assert block4.sha256 not in self.old_node.getdataset @subtest def test_v0_outputs_arent_spendable(self): @@ -537,7 +536,7 @@ class SegWitTest(BitcoinTestFramework): """Mine enough blocks for segwit's vb state to be 'started'.""" height = self.nodes[0].getblockcount() # Will need to rewrite the tests here if we are past the first period - assert(height < VB_PERIOD - 1) + assert height < VB_PERIOD - 1 # Advance to end of period, status should now be 'started' self.nodes[0].generate(VB_PERIOD - height - 1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') @@ -554,19 +553,19 @@ class SegWitTest(BitcoinTestFramework): # If this is a non-segwit node, we should not get a witness # commitment, nor a version bit signalling segwit. assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) - assert('default_witness_commitment' not in gbt_results) + assert 'default_witness_commitment' not in gbt_results else: # For segwit-aware nodes, check the version bit and the witness # commitment are correct. - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - assert('default_witness_commitment' in gbt_results) + assert block_version & (1 << VB_WITNESS_BIT) != 0 + assert 'default_witness_commitment' in gbt_results witness_commitment = gbt_results['default_witness_commitment'] # Check that default_witness_commitment is present. witness_root = CBlock.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, bytes_to_hex_str(script)) + assert_equal(witness_commitment, script.hex()) @subtest def advance_to_segwit_lockin(self): @@ -575,7 +574,7 @@ class SegWitTest(BitcoinTestFramework): # Advance to end of period, and verify lock-in happens at the end self.nodes[0].generate(VB_PERIOD - 1) height = self.nodes[0].getblockcount() - assert((height % VB_PERIOD) == VB_PERIOD - 2) + assert (height % VB_PERIOD) == VB_PERIOD - 2 assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') @@ -600,7 +599,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_message["getdata"].inv[0].type == 1) + assert self.old_node.last_message["getdata"].inv[0].type == 1 # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -646,7 +645,7 @@ class SegWitTest(BitcoinTestFramework): # Mine it on test_node to create the confirmed output. test_transaction_acceptance(self.nodes[0], self.test_node, p2sh_tx, with_witness=True, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Now test standardness of v0 P2WSH outputs. # Start by creating a transaction with two outputs. @@ -677,7 +676,7 @@ class SegWitTest(BitcoinTestFramework): tx3 = CTransaction() # tx and tx2 were both accepted. Don't bother trying to reclaim the # P2PKH output; just send tx's first output back to an anyone-can-spend. - sync_mempools([self.nodes[0], self.nodes[1]]) + self.sync_mempools([self.nodes[0], self.nodes[1]]) tx3.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] tx3.vout = [CTxOut(tx.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))] tx3.wit.vtxinwit.append(CTxInWitness()) @@ -686,17 +685,17 @@ class SegWitTest(BitcoinTestFramework): if self.segwit_status != 'active': # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -734,7 +733,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True, with_witness=True) - sync_blocks(self.nodes) + self.sync_blocks() # Now test attempts to spend the output. spend_tx = CTransaction() @@ -792,7 +791,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # Test the test -- witness serialization should be different - assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) + assert msg_witness_block(block).serialize() != msg_block(block).serialize() # This empty block should be valid. test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -803,7 +802,7 @@ class SegWitTest(BitcoinTestFramework): block_2.solve() # The commitment should have changed! - assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) + assert block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1] # This should also be valid. test_witness_block(self.nodes[0], self.test_node, block_2, accepted=True) @@ -850,7 +849,7 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() block_3.rehash() - assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns + assert len(block_3.vtx[0].vout) == 4 # 3 OP_returns block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=True) @@ -881,19 +880,19 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a' * 5000000) - assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) > MAX_BLOCK_BASE_SIZE # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() != block.hash) + assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() == block.hash) + assert self.nodes[0].getbestblockhash() == block.hash # Now make sure that malleating the witness reserved value doesn't # result in a block permanently marked bad. @@ -918,7 +917,7 @@ class SegWitTest(BitcoinTestFramework): # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() - assert(len(self.utxo) > 0) + assert len(self.utxo) > 0 # Create a P2WSH transaction. # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. @@ -940,7 +939,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_OUTPUTS): parent_tx.vout.append(CTxOut(child_value, script_pubkey)) parent_tx.vout[0].nValue -= 50000 - assert(parent_tx.vout[0].nValue > 0) + assert parent_tx.vout[0].nValue > 0 parent_tx.rehash() child_tx = CTransaction() @@ -970,7 +969,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) # Make sure that our test case would exceed the old max-network-message # limit - assert(len(block.serialize(True)) > 2 * 1024 * 1024) + assert len(block.serialize(True)) > 2 * 1024 * 1024 test_witness_block(self.nodes[0], self.test_node, block, accepted=False) @@ -980,7 +979,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) == MAX_BLOCK_BASE_SIZE test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -998,14 +997,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert(self.nodes[0].getbestblockhash() != block.hash) + self.nodes[0].submitblock(block.serialize(True).hex()) + assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -1020,9 +1019,9 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) + self.nodes[0].submitblock(block_2.serialize(True).hex()) # Tip should not advance! - assert(self.nodes[0].getbestblockhash() != block_2.hash) + assert self.nodes[0].getbestblockhash() != block_2.hash @subtest def test_extra_witness_data(self): @@ -1142,7 +1141,7 @@ class SegWitTest(BitcoinTestFramework): # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) - assert(len(long_witness_program) == MAX_PROGRAM_LENGTH + 1) + assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1166,7 +1165,7 @@ class SegWitTest(BitcoinTestFramework): # Try again with one less byte in the witness program witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) - assert(len(witness_program) == MAX_PROGRAM_LENGTH) + assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) @@ -1197,7 +1196,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(10): tx.vout.append(CTxOut(int(value / 10), script_pubkey)) tx.vout[0].nValue -= 1000 - assert(tx.vout[0].nValue >= 0) + assert tx.vout[0].nValue >= 0 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) @@ -1347,8 +1346,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(raw_tx["vsize"], vsize) assert_equal(raw_tx["weight"], weight) assert_equal(len(raw_tx["vin"][0]["txinwitness"]), 1) - assert_equal(raw_tx["vin"][0]["txinwitness"][0], hexlify(witness_program).decode('ascii')) - assert(vsize != raw_tx["size"]) + assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_program.hex()) + assert vsize != raw_tx["size"] # Cleanup: mine the transactions and update utxo for next test self.nodes[0].generate(1) @@ -1361,7 +1360,8 @@ class SegWitTest(BitcoinTestFramework): def test_segwit_versions(self): """Test validity of future segwit version transactions. - Future segwit version transactions are non-standard, but valid in blocks. + Future segwit versions are non-standard to spend, but valid in blocks. + Sending to future segwit versions is always allowed. Can run this before and after segwit activation.""" NUM_SEGWIT_VERSIONS = 17 # will test OP_0, OP1, ..., OP_16 @@ -1379,7 +1379,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_SEGWIT_VERSIONS): self.utxo.append(UTXO(tx.sha256, i, split_value)) - sync_blocks(self.nodes) + self.sync_blocks() temp_utxo = [] tx = CTransaction() witness_program = CScript([OP_TRUE]) @@ -1397,11 +1397,11 @@ class SegWitTest(BitcoinTestFramework): temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) self.nodes[0].generate(1) # Mine all the transactions - sync_blocks(self.nodes) - assert(len(self.nodes[0].getrawmempool()) == 0) + self.sync_blocks() + assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 1 transactions - # are non-standard + # are standard script_pubkey = CScript([CScriptOp(OP_1), witness_hash]) tx2 = CTransaction() tx2.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] @@ -1409,10 +1409,9 @@ class SegWitTest(BitcoinTestFramework): tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] tx2.rehash() - # Gets accepted to test_node, because standardness of outputs isn't - # checked with fRequireStandard + # Gets accepted to both policy-enforcing nodes and others. test_transaction_acceptance(self.nodes[0], self.test_node, tx2, with_witness=True, accepted=True) - test_transaction_acceptance(self.nodes[1], self.std_node, tx2, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[1], self.std_node, tx2, with_witness=True, accepted=True) temp_utxo.pop() # last entry in temp_utxo was the output we just spent temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -1434,7 +1433,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2, tx3]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() # Add utxo to our list self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) @@ -1462,7 +1461,7 @@ class SegWitTest(BitcoinTestFramework): # Now test a premature spend. self.nodes[0].generate(98) - sync_blocks(self.nodes) + self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=False) @@ -1472,7 +1471,7 @@ class SegWitTest(BitcoinTestFramework): block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() @subtest def test_uncompressed_pubkey(self): @@ -1483,10 +1482,9 @@ class SegWitTest(BitcoinTestFramework): # Segwit transactions using uncompressed pubkeys are not accepted # under default policy, but should still pass consensus. - key = CECKey() - key.set_secretbytes(b"9") - key.set_compressed(False) - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate(False) + pubkey = key.get_pubkey().get_bytes() assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey utxo = self.utxo.pop(0) @@ -1516,7 +1514,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_wsh)) script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[0].scriptWitness.stack = [signature, pubkey] tx2.rehash() @@ -1570,7 +1568,7 @@ class SegWitTest(BitcoinTestFramework): tx5.vin.append(CTxIn(COutPoint(tx4.sha256, 0), b"")) tx5.vout.append(CTxOut(tx4.vout[0].nValue - 1000, CScript([OP_TRUE]))) (sig_hash, err) = SignatureHash(script_pubkey, tx5, 0, SIGHASH_ALL) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx5.vin[0].scriptSig = CScript([signature, pubkey]) tx5.rehash() # Should pass policy and consensus. @@ -1583,9 +1581,9 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_signature_version_1(self): - key = CECKey() - key.set_secretbytes(b"9") - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) witness_hash = sha256(witness_program) @@ -1602,7 +1600,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() self.utxo.pop(0) # Test each hashtype @@ -1667,7 +1665,7 @@ class SegWitTest(BitcoinTestFramework): # Create a slight bias for producing more utxos num_outputs = random.randint(1, 11) random.shuffle(temp_utxos) - assert(len(temp_utxos) > num_inputs) + assert len(temp_utxos) > num_inputs tx = CTransaction() total_value = 0 for i in range(num_inputs): @@ -1720,7 +1718,7 @@ class SegWitTest(BitcoinTestFramework): script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL # Check that we can't have a scriptSig tx2.vin[0].scriptSig = CScript([signature, pubkey]) @@ -1781,7 +1779,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() test_transaction_acceptance(self.nodes[0], self.test_node, tx, False, True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # We'll add an unnecessary witness to this transaction that would cause # it to be non-standard, to test that violating policy with a witness @@ -1810,7 +1808,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx3, False, True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Update our utxo list; we spent the first entry. self.utxo.pop(0) @@ -1846,7 +1844,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Creating transactions for tests p2wsh_txs = [] @@ -1910,7 +1908,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node # Valid but non-standard transactions in a block should be accepted by standard node - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -1925,10 +1923,10 @@ class SegWitTest(BitcoinTestFramework): self.start_node(2, extra_args=["-vbparams=segwit:0:999999999999"]) connect_nodes(self.nodes[0], 2) - sync_blocks(self.nodes) + self.sync_blocks() # Make sure that this peer thinks segwit has activated. - assert(get_bip9_status(self.nodes[2], 'segwit')['status'] == "active") + assert get_bip9_status(self.nodes[2], 'segwit')['status'] == "active" # Make sure this peer's blocks match those of node0. height = self.nodes[2].getblockcount() @@ -1955,7 +1953,7 @@ class SegWitTest(BitcoinTestFramework): extra_sigops_available = MAX_SIGOP_COST % sigops_per_script # We chose the number of checkmultisigs/checksigs to make this work: - assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT + assert extra_sigops_available < 100 # steer clear of MAX_OPS_PER_SCRIPT # This script, when spent with the first # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, @@ -2022,7 +2020,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block_4, accepted=True) # Reset the tip back down for the next test - sync_blocks(self.nodes) + self.sync_blocks() for x in self.nodes: x.invalidateblock(block_4.hash) @@ -2039,5 +2037,31 @@ class SegWitTest(BitcoinTestFramework): # TODO: test p2sh sigop counting + def test_superfluous_witness(self): + # Serialization of tx that puts witness flag to 1 always + def serialize_with_bogus_witness(tx): + flags = 1 + r = b"" + r += struct.pack("<i", tx.nVersion) + if flags: + dummy = [] + r += ser_vector(dummy) + r += struct.pack("<B", flags) + r += ser_vector(tx.vin) + r += ser_vector(tx.vout) + if flags & 1: + if (len(tx.wit.vtxinwit) != len(tx.vin)): + # vtxinwit must have the same length as vin + tx.wit.vtxinwit = tx.wit.vtxinwit[:len(tx.vin)] + for i in range(len(tx.wit.vtxinwit), len(tx.vin)): + tx.wit.vtxinwit.append(CTxInWitness()) + r += tx.wit.serialize() + r += struct.pack("<I", tx.nLockTime) + return r + + raw = self.nodes[0].createrawtransaction([{"txid":"00"*32, "vout":0}], {self.nodes[0].getnewaddress():1}) + tx = FromHex(CTransaction(), raw) + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, serialize_with_bogus_witness(tx).hex()) + if __name__ == '__main__': SegWitTest().main() diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 7d7d251765..161b67e6d0 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. @@ -103,7 +103,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - sync_blocks, wait_until, ) @@ -225,7 +224,7 @@ class SendHeadersTest(BitcoinTestFramework): # make sure all invalidated blocks are node0's self.nodes[0].generatetoaddress(length, self.nodes[0].get_deterministic_priv_key().address) - sync_blocks(self.nodes, wait=0.1) + self.sync_blocks(self.nodes, wait=0.1) for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_block_announcements() @@ -234,7 +233,7 @@ class SendHeadersTest(BitcoinTestFramework): hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) all_hashes = self.nodes[1].generatetoaddress(length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain - sync_blocks(self.nodes, wait=0.1) + self.sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 11299cbc00..534d275c28 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. @@ -57,7 +57,11 @@ from test_framework.blocktools import create_block, create_coinbase, create_tx_w from test_framework.messages import CBlockHeader, CInv, msg_block, msg_headers, msg_inv from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, sync_blocks +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, +) class AcceptBlockTest(BitcoinTestFramework): @@ -114,7 +118,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. @@ -131,7 +135,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h2f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found # But this block should be accepted by node since it has equal work. self.nodes[0].getblock(block_h2f.hash) @@ -150,7 +154,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found self.nodes[0].getblock(block_h3.hash) # But this block should be accepted by node since it has more work. @@ -263,7 +267,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_292.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) @@ -302,7 +306,7 @@ class AcceptBlockTest(BitcoinTestFramework): # 9. Connect node1 to node0 and ensure it is able to sync connect_nodes(self.nodes[0], 1) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) self.log.info("Successfully synced nodes 1 and 0") if __name__ == '__main__': diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 3938ca98dd..acc7cd811c 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index af150a09af..facb05b54c 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. @@ -172,9 +172,9 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(chaintxstats['txcount'], 2) assert_equal(chaintxstats['window_final_block_hash'], b1_hash) assert_equal(chaintxstats['window_block_count'], 0) - assert('window_tx_count' not in chaintxstats) - assert('window_interval' not in chaintxstats) - assert('txrate' not in chaintxstats) + assert 'window_tx_count' not in chaintxstats + assert 'window_interval' not in chaintxstats + assert 'txrate' not in chaintxstats def _test_gettxoutsetinfo(self): node = self.nodes[0] diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 3cc35a7b9a..7abcd71bb8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test transaction signing using the signrawtransaction* RPCs.""" +"""Test multisig RPCs""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, +) import decimal + class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -17,29 +21,40 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def get_keys(self): node0, node1, node2 = self.nodes - self.add = [node1.getnewaddress() for _ in range(self.nkeys)] - self.pub = [node1.getaddressinfo(a)["pubkey"] for a in self.add] - self.priv = [node1.dumpprivkey(a) for a in self.add] + add = [node1.getnewaddress() for _ in range(self.nkeys)] + self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add] + self.priv = [node1.dumpprivkey(a) for a in add] self.final = node2.getnewaddress() def run_test(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes - # 50 BTC each, rest will be 25 BTC each + self.check_addmultisigaddress_errors() + + self.log.info('Generating blocks ...') node0.generate(149) self.sync_all() self.moved = 0 - for self.nkeys in [3,5]: - for self.nsigs in [2,3]: + for self.nkeys in [3, 5]: + for self.nsigs in [2, 3]: for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: self.get_keys() self.do_multisig() self.checkbalances() + def check_addmultisigaddress_errors(self): + self.log.info('Check that addmultisigaddress fails when the private keys are missing') + addresses = [self.nodes[1].getnewaddress(address_type='legacy') for _ in range(2)] + assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) + for a in addresses: + # Importing all addresses should not change the result + self.nodes[0].importaddress(a) + assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) + def checkbalances(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes node0.generate(100) self.sync_all() @@ -49,13 +64,13 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 - total = 149*50 + (height-149-100)*25 + total = 149 * 50 + (height - 149 - 100) * 25 assert bal1 == 0 assert bal2 == self.moved - assert bal0+bal1+bal2 == total + assert bal0 + bal1 + bal2 == total def do_multisig(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) madd = msig["address"] @@ -74,7 +89,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txid = node0.sendtoaddress(madd, 40) tx = node0.getrawtransaction(txid, True) - vout = [v["n"] for v in tx["vout"] if madd in v["scriptPubKey"].get("addresses",[])] + vout = [v["n"] for v in tx["vout"] if madd in v["scriptPubKey"].get("addresses", [])] assert len(vout) == 1 vout = vout[0] scriptPubKey = tx["vout"][vout]["scriptPubKey"]["hex"] @@ -86,16 +101,17 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): outval = value - decimal.Decimal("0.00001000") rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) - rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], prevtxs) + rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs - 1], prevtxs) rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) self.moved += outval - tx = node0.sendrawtransaction(rawtx3["hex"], True) + tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = node0.generate(1)[0] assert tx in node0.getblock(blk)["tx"] txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 940386eee7..01b8cb1854 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" from test_framework.messages import CTransaction, sha256 from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, hex_str_to_bytes from io import BytesIO @@ -81,7 +81,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(multisig_script) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH - multisig_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(multisig_script))) + multisig_script_hash = sha256(hex_str_to_bytes(multisig_script)).hex() assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) # 4) P2SH scriptPubKey @@ -119,7 +119,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(cltv_script) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH - cltv_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(cltv_script))) + cltv_script_hash = sha256(hex_str_to_bytes(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) # 7) P2PK scriptPubKey @@ -196,7 +196,7 @@ class DecodeScriptTest(BitcoinTestFramework): # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). - push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] + push_signature = txSave.vin[0].scriptSig.hex()[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' @@ -206,23 +206,23 @@ class DecodeScriptTest(BitcoinTestFramework): # 1) P2PK scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) def run_test(self): diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 588bfbe083..9a21998d11 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,32 +1,29 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error +# from test_framework.util import assert_raises_rpc_error class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], ["-deprecatedrpc=generate"]] - - def skip_test_if_missing_module(self): - # The generate RPC method requires the wallet to be compiled - self.skip_if_no_wallet() + self.extra_args = [[], []] def run_test(self): # This test should be used to verify correct behaviour of deprecated # RPC methods with and without the -deprecatedrpc flags. For example: # - # self.log.info("Make sure that -deprecatedrpc=createmultisig allows it to take addresses") - # assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, [self.nodes[0].getnewaddress()]) - # self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) - - self.log.info("Test generate RPC") - assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) - self.nodes[1].generate(1) + # In set_test_params: + # self.extra_args = [[], ["-deprecatedrpc=generate"]] + # + # In run_test: + # self.log.info("Test generate RPC") + # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) + # self.nodes[1].generate(1) + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py index 185dffd51c..1984694692 100755 --- a/test/functional/rpc_deriveaddresses.py +++ b/test/functional/rpc_deriveaddresses.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the deriveaddresses rpc call.""" diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 4f350953b2..d89fd6461f 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -94,7 +94,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test that we have enough inputs + assert len(dec_tx['vin']) > 0 #test that we have enough inputs ############################## # simple test with two coins # @@ -107,7 +107,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test if we have enough inputs + assert len(dec_tx['vin']) > 0 #test if we have enough inputs ############################## # simple test with two coins # @@ -120,7 +120,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -139,7 +139,7 @@ class RawTransactionsTest(BitcoinTestFramework): for out in dec_tx['vout']: totalOut += out['value'] - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -363,7 +363,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ ############################################################ @@ -378,7 +378,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -405,7 +405,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -438,7 +438,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -558,7 +558,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance*19) #~19 inputs + assert feeDelta >= 0 and feeDelta <= feeTolerance*19 #~19 inputs ############################################# @@ -620,7 +620,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(res_dec["vin"]), 1) assert_equal(res_dec["vin"][0]["txid"], watchonly_txid) - assert("fee" in result.keys()) + assert "fee" in result.keys() assert_greater_than(result["changepos"], -1) ############################################################### @@ -635,16 +635,16 @@ class RawTransactionsTest(BitcoinTestFramework): result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) - assert(res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid) + assert res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid assert_greater_than(result["fee"], 0) assert_greater_than(result["changepos"], -1) assert_equal(result["fee"] + res_dec["vout"][result["changepos"]]["value"], watchonly_amount / 10) signedtx = self.nodes[3].signrawtransactionwithwallet(result["hex"]) - assert(not signedtx["complete"]) + assert not signedtx["complete"] signedtx = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) - assert(signedtx["complete"]) + assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) self.nodes[0].generate(1) self.sync_all() @@ -676,10 +676,10 @@ class RawTransactionsTest(BitcoinTestFramework): for out in res_dec['vout']: if out['value'] > 1.0: changeaddress += out['scriptPubKey']['addresses'][0] - assert(changeaddress != "") + assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() # Now the change address key should be removed from the keypool - assert(changeaddress != nextaddr) + assert changeaddress != nextaddr ###################################### # Test subtractFeeFromOutputs option # diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py new file mode 100755 index 0000000000..bd93b6f7a4 --- /dev/null +++ b/test/functional/rpc_getblockfilter.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getblockfilter RPC.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, assert_is_hex_string, assert_raises_rpc_error, + connect_nodes, disconnect_nodes, sync_blocks + ) + +FILTER_TYPES = ["basic"] + +class GetBlockFilterTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [["-blockfilterindex"], []] + + def run_test(self): + # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting + disconnect_nodes(self.nodes[0], 1) + + self.nodes[0].generate(3) + self.nodes[1].generate(4) + + assert_equal(self.nodes[0].getblockcount(), 3) + chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Reorg node 0 to a new chain + connect_nodes(self.nodes[0], 1) + sync_blocks(self.nodes) + + assert_equal(self.nodes[0].getblockcount(), 4) + chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Test getblockfilter returns a filter for all blocks and filter types on active chain + for block_hash in chain1_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter returns a filter for all blocks and filter types on stale chain + for block_hash in chain0_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter with unknown block + bad_block_hash = "0123456789abcdef" * 4 + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockfilter, bad_block_hash, "basic") + + # Test getblockfilter with undefined filter type + genesis_hash = self.nodes[0].getblockhash(0) + assert_raises_rpc_error(-5, "Unknown filtertype", self.nodes[0].getblockfilter, genesis_hash, "unknown") + +if __name__ == '__main__': + GetBlockFilterTest().main() diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index feba16f1b3..e17a8f6421 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/rpc_getchaintips.py b/test/functional/rpc_getchaintips.py index c869c7262f..8dc8474374 100755 --- a/test/functional/rpc_getchaintips.py +++ b/test/functional/rpc_getchaintips.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getchaintips RPC. @@ -28,7 +28,8 @@ class GetChainTipsTest (BitcoinTestFramework): self.split_network() self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) self.nodes[2].generatetoaddress(20, self.nodes[2].get_deterministic_priv_key().address) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) tips = self.nodes[1].getchaintips () assert_equal (len (tips), 1) diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index d8a1deb2a3..3d3f694fd3 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE from test_framework.util import ( assert_equal, connect_nodes_bi, - sync_blocks, wait_until, ) @@ -35,7 +34,7 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("Connect nodes to force a reorg") connect_nodes_bi(self.nodes, 0, 1) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) assert_equal(self.nodes[0].getblockcount(), 6) badhash = self.nodes[1].getblockhash(2) @@ -47,7 +46,7 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("Make sure we won't reorg to a lower work chain:") connect_nodes_bi(self.nodes, 1, 2) self.log.info("Sync node 2 to node 1 so both have 6 blocks") - sync_blocks(self.nodes[1:3]) + self.sync_blocks(self.nodes[1:3]) assert_equal(self.nodes[2].getblockcount(), 6) self.log.info("Invalidate block 5 on node 1 so its tip is now at 4") self.nodes[1].invalidateblock(self.nodes[1].getblockhash(5)) diff --git a/test/functional/rpc_named_arguments.py b/test/functional/rpc_named_arguments.py index 6c5c29dfdf..ecac9c2f82 100755 --- a/test/functional/rpc_named_arguments.py +++ b/test/functional/rpc_named_arguments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test using named arguments for RPCs.""" @@ -17,7 +17,7 @@ class NamedArgumentTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] h = node.help(command='getblockchaininfo') - assert(h.startswith('getblockchaininfo\n')) + assert h.startswith('getblockchaininfo\n') assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getblockchaininfo') diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 72e6e6329f..2d5631bb27 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" @@ -8,7 +8,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, - sync_blocks, ) def unidirectional_node_sync_via_rpc(node_src, node_dest): @@ -16,7 +15,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blockhash = node_src.getbestblockhash() while True: try: - assert(len(node_dest.getblock(blockhash, False)) > 0) + assert len(node_dest.getblock(blockhash, False)) > 0 break except: blocks_to_copy.append(blockhash) @@ -24,7 +23,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blocks_to_copy.reverse() for blockhash in blocks_to_copy: blockdata = node_src.getblock(blockhash, False) - assert(node_dest.submitblock(blockdata) in (None, 'inconclusive')) + assert node_dest.submitblock(blockdata) in (None, 'inconclusive') def node_sync_via_rpc(nodes): for node_src in nodes: @@ -57,7 +56,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine competing blocks E-F-G on Node 1") hashG = self.nodes[1].generatetoaddress(3, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 5) - assert(hashC != hashG) + assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) @@ -72,7 +71,7 @@ class PreciousTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbestblockhash(), hashC) self.log.info("Make Node1 prefer block C") self.nodes[1].preciousblock(hashC) - sync_blocks(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC + self.sync_blocks(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC assert_equal(self.nodes[1].getbestblockhash(), hashC) self.log.info("Make Node1 prefer block G again") self.nodes[1].preciousblock(hashG) @@ -86,7 +85,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") self.nodes[0].generatetoaddress(1, gen_address(0)) assert_equal(self.nodes[0].getblockcount(), 6) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() assert_equal(self.nodes[1].getbestblockhash(), hashH) self.log.info("Node1 should not be able to prefer block C anymore") diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index e9098e4f5a..8a7ea7aa58 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. @@ -7,7 +7,13 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, disconnect_nodes, find_output, sync_blocks +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes_bi, + disconnect_nodes, + find_output, +) import json import os @@ -41,22 +47,22 @@ class PSBTTest(BitcoinTestFramework): online_node.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) mining_node.generate(nblocks=1) - sync_blocks([mining_node, online_node]) + self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) utxos = online_node.listunspent(addresses=[offline_addr]) raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] - assert("non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]) + assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] - assert("witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0]) + assert "witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) mining_node.generate(1) - sync_blocks([mining_node, online_node]) + self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) # Reconnect @@ -144,10 +150,11 @@ class PSBTTest(BitcoinTestFramework): new_psbt = self.nodes[0].converttopsbt(rawtx['hex']) self.nodes[0].decodepsbt(new_psbt) - # Make sure that a psbt with signatures cannot be converted + # Make sure that a non-psbt with signatures cannot be converted + # Error could be either "TX decode failed" (segwit inputs causes parsing to fail) or "Inputs must not have scriptSigs and scriptWitnesses" signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex']) - assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex']) - assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex'], False) + assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, signedtx['hex']) + assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, signedtx['hex'], False) # Unless we allow it to convert and strip signatures self.nodes[0].converttopsbt(signedtx['hex'], True) @@ -192,8 +199,8 @@ class PSBTTest(BitcoinTestFramework): psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"replaceable":True}, False) decoded_psbt = self.nodes[0].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): - assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) - assert "bip32_derivs" not in psbt_in + assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) + assert "bip32_derivs" not in psbt_in assert_equal(decoded_psbt["tx"]["locktime"], block_height+2) # Same construction with only locktime set @@ -355,7 +362,7 @@ class PSBTTest(BitcoinTestFramework): assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program'] # Check fee and size things - assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == '0.00746268 BTC/kB' + assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == Decimal('0.00746268') # After signing and finalizing, needs extracting signed = self.nodes[1].walletprocesspsbt(updated)['psbt'] diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index c793cf00de..6e71817dc3 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -17,7 +17,7 @@ from decimal import Decimal from io import BytesIO from test_framework.messages import CTransaction, ToHex from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes class multidict(dict): """Dictionary that allows duplicate keys. @@ -119,21 +119,21 @@ class RawTransactionsTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs={address: 99})))) assert_equal(len(tx.vout), 1) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))))) assert_equal(len(tx.vout), 2) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) # Multiple mixed outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))))) assert_equal(len(tx.vout), 3) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) @@ -285,11 +285,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "amount" : vout['value']}] @@ -330,11 +326,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx2['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx2['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex'], "amount" : vout['value']}] @@ -434,5 +426,30 @@ class RawTransactionsTest(BitcoinTestFramework): decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + self.log.info('sendrawtransaction/testmempoolaccept with maxfeerate') + + txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) + rawTx = self.nodes[0].getrawtransaction(txId, True) + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) + + self.sync_all() + inputs = [{ "txid" : txId, "vout" : vout['n'] }] + outputs = { self.nodes[0].getnewaddress() : Decimal("0.99999000") } # 1000 sat fee + rawTx = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) + assert_equal(rawTxSigned['complete'], True) + # 1000 sat fee, ~200 b transaction, fee rate should land around 5 sat/b = 0.00005000 BTC/kB + # Thus, testmempoolaccept should reject + testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] + assert_equal(testres['allowed'], False) + assert_equal(testres['reject-reason'], '256: absurdly-high-fee') + # and sendrawtransaction should throw + assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + # And below calls should both succeed + testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.00007000')[0] + assert_equal(testres['allowed'], True) + self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.00007000') + + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 18258f5df3..6346477922 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessage.py index ad0e29b451..0cb3ce4215 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing and verifying messages.""" @@ -25,18 +25,18 @@ class SignMessagesTest(BitcoinTestFramework): expected_signature = 'INbVnW4e6PeRmsv2Qgu8NuopvrVjkcxob+sX8OcZG0SALhWybUjzMLPdAsXI46YZGb0KQTRii+wWIQzRpG/U+S0=' signature = self.nodes[0].signmessagewithprivkey(priv_key, message) assert_equal(expected_signature, signature) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test signing with an address with wallet') address = self.nodes[0].getnewaddress() signature = self.nodes[0].signmessage(address, message) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test verifying with another address should not work') other_address = self.nodes[0].getnewaddress() other_signature = self.nodes[0].signmessage(other_address, message) - assert(not self.nodes[0].verifymessage(other_address, signature, message)) - assert(not self.nodes[0].verifymessage(address, other_signature, message)) + assert not self.nodes[0].verifymessage(other_address, signature, message) + assert not self.nodes[0].verifymessage(address, other_signature, message) if __name__ == '__main__': SignMessagesTest().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 56e2c73a90..780758e219 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes from test_framework.messages import sha256 from test_framework.script import CScript, OP_0 @@ -15,7 +15,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-deprecatedrpc=signrawtransaction"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -161,7 +160,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0] assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"]) p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))]) - assert_equal(unspent_output["redeemScript"], bytes_to_hex_str(p2sh_redeemScript)) + assert_equal(unspent_output["redeemScript"], p2sh_redeemScript.hex()) # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")}) spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 20b6341550..e86f91b1d0 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC call related to the uptime command. @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) - assert(self.nodes[0].uptime() >= wait_time) + assert self.nodes[0].uptime() >= wait_time if __name__ == '__main__': diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 456d43aa2e..f36cffe957 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" from .script import hash256, hash160, sha256, CScript, OP_0 -from .util import bytes_to_hex_str, hex_str_to_bytes +from .util import hex_str_to_bytes from . import segwit_addr @@ -16,9 +16,9 @@ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def byte_to_base58(b, version): result = '' - str = bytes_to_hex_str(b) - str = bytes_to_hex_str(chr(version).encode('latin-1')) + str - checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str))) + str = b.hex() + str = chr(version).encode('latin-1').hex() + str + checksum = hash256(hex_str_to_bytes(str)).hex() str += checksum[:8] value = int('0x'+str,0) while value > 0: @@ -32,12 +32,12 @@ def byte_to_base58(b, version): # TODO: def base58_decode def keyhash_to_p2pkh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 0 if main else 111 return byte_to_base58(hash, version) def scripthash_to_p2sh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 5 if main else 196 return byte_to_base58(hash, version) @@ -80,11 +80,11 @@ def check_key(key): key = hex_str_to_bytes(key) # Assuming this is hex string if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): return key - assert(False) + assert False def check_script(script): if (type(script) is str): script = hex_str_to_bytes(script) # Assuming this is hex string if (type(script) is bytes or type(script) is CScript): return script - assert(False) + assert False diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 1140fe9b3e..4ba6ac1db2 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -35,6 +35,7 @@ ServiceProxy class: import base64 import decimal +from http import HTTPStatus import http.client import json import logging @@ -49,13 +50,14 @@ USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): - def __init__(self, rpc_error): + def __init__(self, rpc_error, http_status=None): try: errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' super().__init__(errmsg) self.error = rpc_error + self.http_status = http_status def EncodeDecimal(o): @@ -120,8 +122,11 @@ class AuthServiceProxy(): def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s" % (AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("-{}-> {} {}".format( + AuthServiceProxy.__id_count, + self._service_name, + json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii), + )) if args and argsn: raise ValueError('Cannot handle both named and positional arguments') return {'version': '1.1', @@ -131,19 +136,26 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - response = self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: - raise JSONRPCException(response['error']) + raise JSONRPCException(response['error'], status) elif 'result' not in response: raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}) + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) else: return response['result'] def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) - return self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + return response def _get_response(self): req_start_time = time.time() @@ -162,8 +174,9 @@ class AuthServiceProxy(): content_type = http_response.getheader('Content-Type') if content_type != 'application/json': - raise JSONRPCException({ - 'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}) + raise JSONRPCException( + {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, + http_response.status) responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) @@ -172,7 +185,7 @@ class AuthServiceProxy(): log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) - return response + return response, http_response.status def __truediv__(self, relative_uri): return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 15f4502994..7ac044d0d0 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -20,7 +20,6 @@ from .messages import ( CTxOut, FromHex, ToHex, - bytes_to_hex_str, hash256, hex_str_to_bytes, ser_string, @@ -132,7 +131,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C Can optionally pass scriptPubKey and scriptSig, default is anyone-can-spend output. """ tx = CTransaction() - assert(n < len(prevtx.vout)) + assert n < len(prevtx.vout) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff)) tx.vout.append(CTxOut(amount, script_pub_key)) tx.calc_sha256() @@ -190,7 +189,7 @@ def witness_script(use_p2wsh, pubkey): witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) scripthash = sha256(witness_program) pkscript = CScript([OP_0, scripthash]) - return bytes_to_hex_str(pkscript) + return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): """Return a transaction (in hex) that spends the given utxo to a segwit output. @@ -215,7 +214,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) if (sign): signed = node.signrawtransactionwithwallet(tx_to_witness) - assert("errors" not in signed or len(["errors"]) == 0) + assert "errors" not in signed or len(["errors"]) == 0 return node.sendrawtransaction(signed["hex"]) else: if (insert_redeem_script): diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 1b3e510dc4..912c0ca978 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -1,226 +1,386 @@ -# Copyright (c) 2011 Sam Rushing -"""ECC secp256k1 OpenSSL wrapper. +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test-only secp256k1 elliptic curve implementation -WARNING: This module does not mlock() secrets; your private keys may end up on -disk in swap! Use with caution! +WARNING: This code is slow, uses bad randomness, does not properly protect +keys, and is trivially vulnerable to side channel attacks. Do not use for +anything but tests.""" +import random -This file is modified from python-bitcoinlib. -""" - -import ctypes -import ctypes.util -import hashlib - -ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library ('ssl') or 'libeay32') - -ssl.BN_new.restype = ctypes.c_void_p -ssl.BN_new.argtypes = [] - -ssl.BN_bin2bn.restype = ctypes.c_void_p -ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] - -ssl.BN_CTX_free.restype = None -ssl.BN_CTX_free.argtypes = [ctypes.c_void_p] - -ssl.BN_CTX_new.restype = ctypes.c_void_p -ssl.BN_CTX_new.argtypes = [] - -ssl.ECDH_compute_key.restype = ctypes.c_int -ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_sign.restype = ctypes.c_int -ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_verify.restype = ctypes.c_int -ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] - -ssl.EC_KEY_free.restype = None -ssl.EC_KEY_free.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int] - -ssl.EC_KEY_get0_group.restype = ctypes.c_void_p -ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p -ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_set_private_key.restype = ctypes.c_int -ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_KEY_set_conv_form.restype = None -ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int] - -ssl.EC_KEY_set_public_key.restype = ctypes.c_int -ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.i2o_ECPublicKey.restype = ctypes.c_void_p -ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_POINT_new.restype = ctypes.c_void_p -ssl.EC_POINT_new.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_free.restype = None -ssl.EC_POINT_free.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_mul.restype = ctypes.c_int -ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -# this specifies the curve used with ECDSA. -NID_secp256k1 = 714 # from openssl/obj_mac.h +def modinv(a, n): + """Compute the modular inverse of a modulo n + See https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm#Modular_integers. + """ + t1, t2 = 0, 1 + r1, r2 = n, a + while r2 != 0: + q = r1 // r2 + t1, t2 = t2, t1 - q * t2 + r1, r2 = r2, r1 - q * r2 + if r1 > 1: + return None + if t1 < 0: + t1 += n + return t1 + +def jacobi_symbol(n, k): + """Compute the Jacobi symbol of n modulo k + + See http://en.wikipedia.org/wiki/Jacobi_symbol + + For our application k is always prime, so this is the same as the Legendre symbol.""" + assert k > 0 and k & 1, "jacobi symbol is only defined for positive odd k" + n %= k + t = 0 + while n != 0: + while n & 1 == 0: + n >>= 1 + r = k & 7 + t ^= (r == 3 or r == 5) + n, k = k, n + t ^= (n & k & 3 == 3) + n = n % k + if k == 1: + return -1 if t else 1 + return 0 + +def modsqrt(a, p): + """Compute the square root of a modulo p when p % 4 = 3. + + The Tonelli-Shanks algorithm can be used. See https://en.wikipedia.org/wiki/Tonelli-Shanks_algorithm + + Limiting this function to only work for p % 4 = 3 means we don't need to + iterate through the loop. The highest n such that p - 1 = 2^n Q with Q odd + is n = 1. Therefore Q = (p-1)/2 and sqrt = a^((Q+1)/2) = a^((p+1)/4) + + secp256k1's is defined over field of size 2**256 - 2**32 - 977, which is 3 mod 4. + """ + if p % 4 != 3: + raise NotImplementedError("modsqrt only implemented for p % 4 = 3") + sqrt = pow(a, (p + 1)//4, p) + if pow(sqrt, 2, p) == a % p: + return sqrt + return None + +class EllipticCurve: + def __init__(self, p, a, b): + """Initialize elliptic curve y^2 = x^3 + a*x + b over GF(p).""" + self.p = p + self.a = a % p + self.b = b % p + + def affine(self, p1): + """Convert a Jacobian point tuple p1 to affine form, or None if at infinity. + + An affine point is represented as the Jacobian (x, y, 1)""" + x1, y1, z1 = p1 + if z1 == 0: + return None + inv = modinv(z1, self.p) + inv_2 = (inv**2) % self.p + inv_3 = (inv_2 * inv) % self.p + return ((inv_2 * x1) % self.p, (inv_3 * y1) % self.p, 1) + + def negate(self, p1): + """Negate a Jacobian point tuple p1.""" + x1, y1, z1 = p1 + return (x1, (self.p - y1) % self.p, z1) + + def on_curve(self, p1): + """Determine whether a Jacobian tuple p is on the curve (and not infinity)""" + x1, y1, z1 = p1 + z2 = pow(z1, 2, self.p) + z4 = pow(z2, 2, self.p) + return z1 != 0 and (pow(x1, 3, self.p) + self.a * x1 * z4 + self.b * z2 * z4 - pow(y1, 2, self.p)) % self.p == 0 + + def is_x_coord(self, x): + """Test whether x is a valid X coordinate on the curve.""" + x_3 = pow(x, 3, self.p) + return jacobi_symbol(x_3 + self.a * x + self.b, self.p) != -1 + + def lift_x(self, x): + """Given an X coordinate on the curve, return a corresponding affine point.""" + x_3 = pow(x, 3, self.p) + v = x_3 + self.a * x + self.b + y = modsqrt(v, self.p) + if y is None: + return None + return (x, y, 1) + + def double(self, p1): + """Double a Jacobian tuple p1 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Doubling""" + x1, y1, z1 = p1 + if z1 == 0: + return (0, 1, 0) + y1_2 = (y1**2) % self.p + y1_4 = (y1_2**2) % self.p + x1_2 = (x1**2) % self.p + s = (4*x1*y1_2) % self.p + m = 3*x1_2 + if self.a: + m += self.a * pow(z1, 4, self.p) + m = m % self.p + x2 = (m**2 - 2*s) % self.p + y2 = (m*(s - x2) - 8*y1_4) % self.p + z2 = (2*y1*z1) % self.p + return (x2, y2, z2) + + def add_mixed(self, p1, p2): + """Add a Jacobian tuple p1 and an affine tuple p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition (with affine point)""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + assert(z2 == 1) + # Adding to the point at infinity is a no-op + if z1 == 0: + return p2 + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + u2 = (x2 * z1_2) % self.p + s2 = (y2 * z1_3) % self.p + if x1 == u2: + if (y1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - x1 + r = s2 - y1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (x1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - y1*h_3) % self.p + z3 = (h*z1) % self.p + return (x3, y3, z3) + + def add(self, p1, p2): + """Add two Jacobian tuples p1 and p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + # Adding the point at infinity is a no-op + if z1 == 0: + return p2 + if z2 == 0: + return p1 + # Adding an Affine to a Jacobian is more efficient since we save field multiplications and squarings when z = 1 + if z1 == 1: + return self.add_mixed(p2, p1) + if z2 == 1: + return self.add_mixed(p1, p2) + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + z2_2 = (z2**2) % self.p + z2_3 = (z2_2 * z2) % self.p + u1 = (x1 * z2_2) % self.p + u2 = (x2 * z1_2) % self.p + s1 = (y1 * z2_3) % self.p + s2 = (y2 * z1_3) % self.p + if u1 == u2: + if (s1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - u1 + r = s2 - s1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (u1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - s1*h_3) % self.p + z3 = (h*z1*z2) % self.p + return (x3, y3, z3) + + def mul(self, ps): + """Compute a (multi) point multiplication + + ps is a list of (Jacobian tuple, scalar) pairs. + """ + r = (0, 1, 0) + for i in range(255, -1, -1): + r = self.double(r) + for (p, n) in ps: + if ((n >> i) & 1): + r = self.add(r, p) + return r + +SECP256K1 = EllipticCurve(2**256 - 2**32 - 977, 0, 7) +SECP256K1_G = (0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8, 1) SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2 -# Thx to Sam Devlin for the ctypes magic 64-bit fix. -def _check_result(val, func, args): - if val == 0: - raise ValueError - else: - return ctypes.c_void_p (val) - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.errcheck = _check_result - -class CECKey(): - """Wrapper around OpenSSL's EC_KEY""" - - POINT_CONVERSION_COMPRESSED = 2 - POINT_CONVERSION_UNCOMPRESSED = 4 +class ECPubKey(): + """A secp256k1 public key""" def __init__(self): - self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1) - - def __del__(self): - if ssl: - ssl.EC_KEY_free(self.k) - self.k = None - - def set_secretbytes(self, secret): - priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new()) - group = ssl.EC_KEY_get0_group(self.k) - pub_key = ssl.EC_POINT_new(group) - ctx = ssl.BN_CTX_new() - if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx): - raise ValueError("Could not derive public key from the supplied secret.") - ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx) - ssl.EC_KEY_set_private_key(self.k, priv_key) - ssl.EC_KEY_set_public_key(self.k, pub_key) - ssl.EC_POINT_free(pub_key) - ssl.BN_CTX_free(ctx) - return self.k - - def set_privkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def set_pubkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def get_privkey(self): - size = ssl.i2d_ECPrivateKey(self.k, 0) - mb_pri = ctypes.create_string_buffer(size) - ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri))) - return mb_pri.raw - - def get_pubkey(self): - size = ssl.i2o_ECPublicKey(self.k, 0) - mb = ctypes.create_string_buffer(size) - ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb))) - return mb.raw - - def get_raw_ecdh_key(self, other_pubkey): - ecdh_keybuffer = ctypes.create_string_buffer(32) - r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32, - ssl.EC_KEY_get0_public_key(other_pubkey.k), - self.k, 0) - if r != 32: - raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed') - return ecdh_keybuffer.raw - - def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()): - # FIXME: be warned it's not clear what the kdf should be as a default - r = self.get_raw_ecdh_key(other_pubkey) - return kdf(r) - - def sign(self, hash, low_s = True): - # FIXME: need unit tests for below cases - if not isinstance(hash, bytes): - raise TypeError('Hash must be bytes instance; got %r' % hash.__class__) - if len(hash) != 32: - raise ValueError('Hash must be exactly 32 bytes long') - - sig_size0 = ctypes.c_uint32() - sig_size0.value = ssl.ECDSA_size(self.k) - mb_sig = ctypes.create_string_buffer(sig_size0.value) - result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k) - assert 1 == result - assert mb_sig.raw[0] == 0x30 - assert mb_sig.raw[1] == sig_size0.value - 2 - total_size = mb_sig.raw[1] - assert mb_sig.raw[2] == 2 - r_size = mb_sig.raw[3] - assert mb_sig.raw[4 + r_size] == 2 - s_size = mb_sig.raw[5 + r_size] - s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big') - if (not low_s) or s_value <= SECP256K1_ORDER_HALF: - return mb_sig.raw[:sig_size0.value] - else: - low_s_value = SECP256K1_ORDER - s_value - low_s_bytes = (low_s_value).to_bytes(33, byteorder='big') - while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80: - low_s_bytes = low_s_bytes[1:] - new_s_size = len(low_s_bytes) - new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big') - new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big') - return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes - - def verify(self, hash, sig): - """Verify a DER signature""" - return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1 - - def set_compressed(self, compressed): - if compressed: - form = self.POINT_CONVERSION_COMPRESSED + """Construct an uninitialized public key""" + self.valid = False + + def set(self, data): + """Construct a public key from a serialization in compressed or uncompressed format""" + if (len(data) == 65 and data[0] == 0x04): + p = (int.from_bytes(data[1:33], 'big'), int.from_bytes(data[33:65], 'big'), 1) + self.valid = SECP256K1.on_curve(p) + if self.valid: + self.p = p + self.compressed = False + elif (len(data) == 33 and (data[0] == 0x02 or data[0] == 0x03)): + x = int.from_bytes(data[1:33], 'big') + if SECP256K1.is_x_coord(x): + p = SECP256K1.lift_x(x) + # if the oddness of the y co-ord isn't correct, find the other + # valid y + if (p[1] & 1) != (data[0] & 1): + p = SECP256K1.negate(p) + self.p = p + self.valid = True + self.compressed = True + else: + self.valid = False else: - form = self.POINT_CONVERSION_UNCOMPRESSED - ssl.EC_KEY_set_conv_form(self.k, form) - + self.valid = False -class CPubKey(bytes): - """An encapsulated public key - - Attributes: + @property + def is_compressed(self): + return self.compressed - is_valid - Corresponds to CPubKey.IsValid() - is_fullyvalid - Corresponds to CPubKey.IsFullyValid() - is_compressed - Corresponds to CPubKey.IsCompressed() - """ + @property + def is_valid(self): + return self.valid + + def get_bytes(self): + assert(self.valid) + p = SECP256K1.affine(self.p) + if p is None: + return None + if self.compressed: + return bytes([0x02 + (p[1] & 1)]) + p[0].to_bytes(32, 'big') + else: + return bytes([0x04]) + p[0].to_bytes(32, 'big') + p[1].to_bytes(32, 'big') + + def verify_ecdsa(self, sig, msg, low_s=True): + """Verify a strictly DER-encoded ECDSA signature against this pubkey. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA verifier algorithm""" + assert(self.valid) + + # Extract r and s from the DER formatted signature. Return false for + # any DER encoding errors. + if (sig[1] + 2 != len(sig)): + return False + if (len(sig) < 4): + return False + if (sig[0] != 0x30): + return False + if (sig[2] != 0x02): + return False + rlen = sig[3] + if (len(sig) < 6 + rlen): + return False + if rlen < 1 or rlen > 33: + return False + if sig[4] >= 0x80: + return False + if (rlen > 1 and (sig[4] == 0) and not (sig[5] & 0x80)): + return False + r = int.from_bytes(sig[4:4+rlen], 'big') + if (sig[4+rlen] != 0x02): + return False + slen = sig[5+rlen] + if slen < 1 or slen > 33: + return False + if (len(sig) != 6 + rlen + slen): + return False + if sig[6+rlen] >= 0x80: + return False + if (slen > 1 and (sig[6+rlen] == 0) and not (sig[7+rlen] & 0x80)): + return False + s = int.from_bytes(sig[6+rlen:6+rlen+slen], 'big') + + # Verify that r and s are within the group order + if r < 1 or s < 1 or r >= SECP256K1_ORDER or s >= SECP256K1_ORDER: + return False + if low_s and s >= SECP256K1_ORDER_HALF: + return False + z = int.from_bytes(msg, 'big') + + # Run verifier algorithm on r, s + w = modinv(s, SECP256K1_ORDER) + u1 = z*w % SECP256K1_ORDER + u2 = r*w % SECP256K1_ORDER + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, u1), (self.p, u2)])) + if R is None or R[0] != r: + return False + return True + +class ECKey(): + """A secp256k1 private key""" - def __new__(cls, buf, _cec_key=None): - self = super(CPubKey, cls).__new__(cls, buf) - if _cec_key is None: - _cec_key = CECKey() - self._cec_key = _cec_key - self.is_fullyvalid = _cec_key.set_pubkey(self) != 0 - return self + def __init__(self): + self.valid = False + + def set(self, secret, compressed): + """Construct a private key object with given 32-byte secret and compressed flag.""" + assert(len(secret) == 32) + secret = int.from_bytes(secret, 'big') + self.valid = (secret > 0 and secret < SECP256K1_ORDER) + if self.valid: + self.secret = secret + self.compressed = compressed + + def generate(self, compressed=True): + """Generate a random private key (compressed or uncompressed).""" + self.set(random.randrange(1, SECP256K1_ORDER).to_bytes(32, 'big'), compressed) + + def get_bytes(self): + """Retrieve the 32-byte representation of this key.""" + assert(self.valid) + return self.secret.to_bytes(32, 'big') @property def is_valid(self): - return len(self) > 0 + return self.valid @property def is_compressed(self): - return len(self) == 33 - - def verify(self, hash, sig): - return self._cec_key.verify(hash, sig) - - def __str__(self): - return repr(self) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__()) + return self.compressed + def get_pubkey(self): + """Compute an ECPubKey object for this secret key.""" + assert(self.valid) + ret = ECPubKey() + p = SECP256K1.mul([(SECP256K1_G, self.secret)]) + ret.p = p + ret.valid = True + ret.compressed = self.compressed + return ret + + def sign_ecdsa(self, msg, low_s=True): + """Construct a DER-encoded ECDSA signature with this key. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA signer algorithm.""" + assert(self.valid) + z = int.from_bytes(msg, 'big') + # Note: no RFC6979, but a simple random nonce (some tests rely on distinct transactions for the same operation) + k = random.randrange(1, SECP256K1_ORDER) + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, k)])) + r = R[0] % SECP256K1_ORDER + s = (modinv(k, SECP256K1_ORDER) * (z + self.secret * r)) % SECP256K1_ORDER + if low_s and s > SECP256K1_ORDER_HALF: + s = SECP256K1_ORDER - s + # Represent in DER format. The byte representations of r and s have + # length rounded up (255 bits becomes 32 bytes and 256 bits becomes 33 + # bytes). + rb = r.to_bytes((r.bit_length() + 8) // 8, 'big') + sb = s.to_bytes((s.bit_length() + 8) // 8, 'big') + return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4bd58519c5..7cf51d9223 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -28,7 +28,7 @@ import struct import time from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, assert_equal +from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -181,7 +181,7 @@ def FromHex(obj, hex_string): # Convert a binary-serializable object to hex (eg for submission via RPC) def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) + return obj.serialize().hex() # Objects that map to bitcoind objects, which can be serialized/deserialized @@ -319,7 +319,7 @@ class CTxIn: def __repr__(self): return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + % (repr(self.prevout), self.scriptSig.hex(), self.nSequence) @@ -343,7 +343,7 @@ class CTxOut: def __repr__(self): return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) + self.scriptPubKey.hex()) class CScriptWitness: @@ -355,7 +355,7 @@ class CScriptWitness: def __repr__(self): return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) + (",".join([x.hex() for x in self.stack])) def is_null(self): if self.stack: diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ac7cc068bd..11ea968257 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -171,7 +171,7 @@ class P2PConnection(asyncio.Protocol): if len(self.recvbuf) < 4: return if self.recvbuf[:4] != self.magic_bytes: - raise ValueError("got garbage %s" % repr(self.recvbuf)) + raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] @@ -218,10 +218,7 @@ class P2PConnection(asyncio.Protocol): def maybe_write(): if not self._transport: return - # Python <3.4.4 does not have is_closing, so we have to check for - # its existence explicitly as long as Bitcoin Core supports all - # Python 3.4 versions. - if hasattr(self._transport, 'is_closing') and self._transport.is_closing(): + if self._transport.is_closing(): return self._transport.write(raw_message_bytes) NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write) @@ -534,7 +531,7 @@ class P2PDataStore(P2PInterface): for b in blocks: self.send_message(msg_block(block=b)) else: - self.send_message(msg_headers([CBlockHeader(blocks[-1])])) + self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) if expect_disconnect: diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 45c182f630..c98424e8e2 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. @@ -12,7 +12,7 @@ import socket import struct import array import os -from binascii import unhexlify, hexlify +from binascii import unhexlify # STATE_ESTABLISHED = '01' # STATE_SYN_SENT = '02' @@ -129,17 +129,17 @@ def addr_to_hex(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr = sub[0] + ([0] * nullbytes) + sub[1] else: raise ValueError('Could not parse address %s' % addr) - return hexlify(bytearray(addr)).decode('ascii') + return bytearray(addr).hex() def test_ipv6_local(): ''' diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 012c80a1be..384062b808 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as SignatureHash(). @@ -9,7 +9,6 @@ This file is modified from python-bitcoinlib. from .messages import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string -from binascii import hexlify import hashlib import struct @@ -450,10 +449,6 @@ class CScript(bytes): # join makes no sense for a CScript() raise NotImplementedError - # Python 3.4 compatibility - def hex(self): - return hexlify(self).decode('ascii') - def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): return super(CScript, cls).__new__(cls, value) @@ -545,7 +540,7 @@ class CScript(bytes): def __repr__(self): def _repr(o): if isinstance(o, bytes): - return "x('%s')" % hexlify(o).decode('ascii') + return "x('%s')" % o.hex() else: return repr(o) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index a21c864e75..799b1c74b8 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" @@ -144,7 +144,7 @@ class Socks5Server(): thread.start() def start(self): - assert(not self.running) + assert not self.running self.running = True self.thread = threading.Thread(None, self.run) self.thread.daemon = True diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 09d7d877a7..555d55d97f 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -396,7 +396,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """ disconnect_nodes(self.nodes[1], 2) disconnect_nodes(self.nodes[2], 1) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) def join_network(self): """ @@ -405,13 +406,15 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): connect_nodes_bi(self.nodes, 1, 2) self.sync_all() - def sync_all(self, node_groups=None): - if not node_groups: - node_groups = [self.nodes] + def sync_blocks(self, nodes=None, **kwargs): + sync_blocks(nodes or self.nodes, **kwargs) - for group in node_groups: - sync_blocks(group) - sync_mempools(group) + def sync_mempools(self, nodes=None, **kwargs): + sync_mempools(nodes or self.nodes, **kwargs) + + def sync_all(self, nodes=None, **kwargs): + self.sync_blocks(nodes, **kwargs) + self.sync_mempools(nodes, **kwargs) # Private helper methods. These should not be accessed by the subclass test scripts. @@ -497,7 +500,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # see the tip age check in IsInitialBlockDownload(). for i in range(8): self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address) - sync_blocks(self.nodes) + self.sync_blocks() for n in self.nodes: assert_equal(n.getblockchaininfo()["blocks"], 199) @@ -553,21 +556,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_CLI") + return self.config["components"].getboolean("ENABLE_CLI") def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_WALLET") + return self.config["components"].getboolean("ENABLE_WALLET") def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_ZMQ") + return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 37fd2a8744..8b2006a05c 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -31,9 +31,6 @@ from .util import ( p2p_port, ) -# For Python 3.4 compatibility -JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) - BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -372,7 +369,7 @@ class TestNode(): stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0 if not sys.platform.startswith('linux'): - self.log.warning("Can't profile with perf; only availabe on Linux platforms") + self.log.warning("Can't profile with perf; only available on Linux platforms") return None if not test_success('which perf'): @@ -565,5 +562,5 @@ class TestNodeCLI(): raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except JSONDecodeError: + except json.JSONDecodeError: return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index fef9982412..190301b215 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" from base64 import b64encode -from binascii import hexlify, unhexlify +from binascii import unhexlify from decimal import Decimal, ROUND_DOWN import hashlib import inspect @@ -182,9 +182,6 @@ def check_json_precision(): def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def bytes_to_hex_str(byte_str): - return hexlify(byte_str).decode('ascii') - def hash256(byte_str): sha256 = hashlib.sha256() sha256.update(byte_str) @@ -267,7 +264,7 @@ def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): - assert(n <= MAX_NODES) + assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): @@ -326,12 +323,14 @@ def get_auth_cookie(datadir): if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") - if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")) and os.access(os.path.join(datadir, "regtest", ".cookie"), os.R_OK): + try: with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] password = split_userpass[1] + except OSError: + pass if user is None or password is None: raise ValueError("No RPC credentials") return user, password @@ -425,7 +424,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ - assert(confirmations_required >= 0) + assert confirmations_required >= 0 utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] @@ -470,7 +469,7 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransactionwithwallet(rawtx) - txid = from_node.sendrawtransaction(signresult["hex"], True) + txid = from_node.sendrawtransaction(signresult["hex"], 0) return (txid, signresult["hex"], fee) @@ -503,7 +502,7 @@ def create_confirmed_utxos(fee, node, count): node.generate(1) utxos = node.listunspent() - assert(len(utxos) >= count) + assert len(utxos) >= count return utxos # Create large OP_RETURN txouts that can be appended to a transaction @@ -542,7 +541,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): newtx = newtx + txouts newtx = newtx + rawtx[94:] signresult = node.signrawtransactionwithwallet(newtx, None, "NONE") - txid = node.sendrawtransaction(signresult["hex"], True) + txid = node.sendrawtransaction(signresult["hex"], 0) txids.append(txid) return txids diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index bb9ac58e46..ec5d6f1715 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -66,6 +66,13 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 +EXTENDED_SCRIPTS = [ + # These tests are not run by the travis build process. + # Longest test should go first, to favor running tests in parallel + 'feature_pruning.py', + 'feature_dbcrash.py', +] + BASE_SCRIPTS = [ # Scripts that are run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -144,6 +151,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', + 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', 'feature_rbf.py', 'mempool_packages.py', @@ -195,13 +203,6 @@ BASE_SCRIPTS = [ # Put them in a random line within the section that fits their approximate run-time ] -EXTENDED_SCRIPTS = [ - # These tests are not run by the travis build process. - # Longest test should go first, to favor running tests in parallel - 'feature_pruning.py', - 'feature_dbcrash.py', -] - # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS @@ -478,6 +479,11 @@ class TestHandler: log_stderr)) if not self.jobs: raise IndexError('pop from empty list') + + # Print remaining running jobs when all jobs have been started. + if not self.test_list: + print("Remaining jobs: [{}]".format(", ".join(j[0] for j in self.jobs))) + dot_count = 0 while True: # Return first proc that finishes diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index b0b151d2d6..e86679bc31 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -18,8 +18,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, - sync_blocks, - sync_mempools, ) @@ -33,12 +31,12 @@ class AbandonConflictTest(BitcoinTestFramework): def run_test(self): self.nodes[1].generate(100) - sync_blocks(self.nodes) + self.sync_blocks() balance = self.nodes[0].getbalance() txA = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txB = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txC = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[1].generate(1) # Can not abandon non-wallet transaction @@ -46,7 +44,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Can not abandon confirmed transaction assert_raises_rpc_error(-5, 'Transaction not eligible for abandonment', lambda: self.nodes[0].abandontransaction(txid=txA)) - sync_blocks(self.nodes) + self.sync_blocks() newbalance = self.nodes[0].getbalance() assert balance - newbalance < Decimal("0.001") #no more than fees lost balance = newbalance @@ -163,7 +161,7 @@ class AbandonConflictTest(BitcoinTestFramework): self.nodes[1].generate(1) connect_nodes(self.nodes[0], 1) - sync_blocks(self.nodes) + self.sync_blocks() # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted newbalance = self.nodes[0].getbalance() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 380dd9c0ad..a40613dfc7 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -63,8 +63,6 @@ from test_framework.util import ( assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, - sync_blocks, - sync_mempools, ) @@ -102,62 +100,62 @@ class AddressTypeTest(BitcoinTestFramework): def test_address(self, node, address, multisig, typ): """Run sanity checks on an address.""" info = self.nodes[node].getaddressinfo(address) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] assert_equal(info.get('solvable'), True) if not multisig and typ == 'legacy': # P2PKH - assert(not info['isscript']) - assert(not info['iswitness']) - assert('pubkey' in info) + assert not info['isscript'] + assert not info['iswitness'] + assert 'pubkey' in info elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert(info['isscript']) - assert(not info['iswitness']) + assert info['isscript'] + assert not info['iswitness'] assert_equal(info['script'], 'witness_v0_keyhash') - assert('pubkey' in info) + assert 'pubkey' in info elif not multisig and typ == 'bech32': # P2WPKH - assert(not info['isscript']) - assert(info['iswitness']) + assert not info['isscript'] + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 40) - assert('pubkey' in info) + assert 'pubkey' in info elif typ == 'legacy': # P2SH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(not info['iswitness']) - assert('pubkeys' in info) + assert not info['iswitness'] + assert 'pubkeys' in info elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'witness_v0_scripthash') - assert(not info['iswitness']) - assert(info['embedded']['isscript']) + assert not info['iswitness'] + assert info['embedded']['isscript'] assert_equal(info['embedded']['script'], 'multisig') - assert(info['embedded']['iswitness']) + assert info['embedded']['iswitness'] assert_equal(info['embedded']['witness_version'], 0) assert_equal(len(info['embedded']['witness_program']), 64) - assert('pubkeys' in info['embedded']) + assert 'pubkeys' in info['embedded'] elif typ == 'bech32': # P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(info['iswitness']) + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 64) - assert('pubkeys' in info) + assert 'pubkeys' in info else: # Unknown type - assert(False) + assert False def test_desc(self, node, address, multisig, typ, utxo): """Run sanity checks on a descriptor reported by getaddressinfo.""" info = self.nodes[node].getaddressinfo(address) - assert('desc' in info) + assert 'desc' in info assert_equal(info['desc'], utxo['desc']) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] # Use a ridiculously roundabout way to find the key origin info through # the PSBT logic. However, this does test consistency between the PSBT reported @@ -172,11 +170,11 @@ class AddressTypeTest(BitcoinTestFramework): key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] # Verify the descriptor checksum against the Python implementation - assert(descsum_check(info['desc'])) + assert descsum_check(info['desc']) # Verify that stripping the checksum and recreating it using Python roundtrips - assert(info['desc'] == descsum_create(info['desc'][:-9])) + assert info['desc'] == descsum_create(info['desc'][:-9]) # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips - assert(info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor']) + assert info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor'] if not multisig and typ == 'legacy': # P2PKH @@ -198,7 +196,7 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) else: # Unknown type - assert(False) + assert False def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) @@ -220,7 +218,7 @@ class AddressTypeTest(BitcoinTestFramework): # Mine 101 blocks on node5 to bring nodes out of IBD and make sure that # no coinbases are maturing for the nodes-under-test during the test self.nodes[5].generate(101) - sync_blocks(self.nodes) + self.sync_blocks() uncompressed_1 = "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee" uncompressed_2 = "047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77" @@ -287,7 +285,7 @@ class AddressTypeTest(BitcoinTestFramework): self.log.debug("Sending: {}".format(sends)) self.nodes[from_node].sendmany("", sends) - sync_mempools(self.nodes) + self.sync_mempools() unconf_balances = self.get_balances(False) self.log.debug("Check unconfirmed balances: {}".format(unconf_balances)) @@ -298,7 +296,7 @@ class AddressTypeTest(BitcoinTestFramework): # node5 collects fee and block subsidy to keep accounting simple self.nodes[5].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor for n, to_node in enumerate(range(from_node, from_node + 4)): @@ -328,7 +326,7 @@ class AddressTypeTest(BitcoinTestFramework): # Fund node 4: self.nodes[5].sendtoaddress(self.nodes[4].getnewaddress(), Decimal("1")) self.nodes[5].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[4].getbalance(), 1) self.log.info("Nodes with addresstype=legacy never use a P2WPKH change output") diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index dd3750203a..55c517e92f 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. @@ -36,7 +36,12 @@ from random import randint import shutil from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, +) + class WalletBackupTest(BitcoinTestFramework): def set_test_params(self): @@ -75,9 +80,9 @@ class WalletBackupTest(BitcoinTestFramework): # Have the miner (node3) mine a block. # Must sync mempools before mining. - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[3].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # As above, this mirrors the original bash test. def start_three(self): @@ -102,13 +107,13 @@ class WalletBackupTest(BitcoinTestFramework): def run_test(self): self.log.info("Generating initial blockchain") self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[1].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[2].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[3].generate(100) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) @@ -165,7 +170,7 @@ class WalletBackupTest(BitcoinTestFramework): self.log.info("Re-starting nodes") self.start_three() - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) @@ -189,7 +194,7 @@ class WalletBackupTest(BitcoinTestFramework): self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 05c97e0340..c02d7422b9 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -1,23 +1,26 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" from decimal import Decimal +import struct +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, + connect_nodes_bi, + sync_blocks, ) -RANDOM_COINBASE_ADDRESS = 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ' def create_transactions(node, address, amt, fees): # Create and sign raw transactions from node to address for amt. # Creates a transaction for each fee and returns an array # of the raw transactions. - utxos = node.listunspent(0) + utxos = [u for u in node.listunspent(0) if u['spendable']] # Create transactions inputs = [] @@ -25,7 +28,7 @@ def create_transactions(node, address, amt, fees): for utxo in utxos: inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]}) ins_total += utxo['amount'] - if ins_total > amt: + if ins_total + max(fees) > amt: break txs = [] @@ -33,6 +36,7 @@ def create_transactions(node, address, amt, fees): outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee} raw_tx = node.createrawtransaction(inputs, outputs, 0, True) raw_tx = node.signrawtransactionwithwallet(raw_tx) + assert_equal(raw_tx['complete'], True) txs.append(raw_tx) return txs @@ -41,21 +45,26 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + self.extra_args = [ + ['-limitdescendantcount=3'], # Limit mempool descendants as a hack to have wallet txs rejected from the mempool + [], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.nodes[0].importaddress(ADDRESS_WATCHONLY) # Check that nodes don't own any UTXOs assert_equal(len(self.nodes[0].listunspent()), 0) assert_equal(len(self.nodes[1].listunspent()), 0) - self.log.info("Mining one block for each node") + self.log.info("Mining blocks ...") self.nodes[0].generate(1) self.sync_all() self.nodes[1].generate(1) - self.nodes[1].generatetoaddress(100, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(101, ADDRESS_WATCHONLY) self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -64,8 +73,10 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance with different arguments") assert_equal(self.nodes[0].getbalance("*"), 50) assert_equal(self.nodes[0].getbalance("*", 1), 50) - assert_equal(self.nodes[0].getbalance("*", 1, True), 50) + assert_equal(self.nodes[0].getbalance("*", 1, True), 100) assert_equal(self.nodes[0].getbalance(minconf=1), 50) + assert_equal(self.nodes[0].getbalance(minconf=0, include_watchonly=True), 100) + assert_equal(self.nodes[1].getbalance(minconf=0, include_watchonly=True), 50) # Send 40 BTC from 0 to 1 and 60 BTC from 1 to 0. txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')]) @@ -83,32 +94,34 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") - # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions - assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('29.99')) # change from node 1's send - # Same with minconf=0 - assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99')) - # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago - # TODO: fix getbalance tracking of coin spentness depth - assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) - assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) - # getunconfirmedbalance - assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + def test_balances(*, fee_node_1=0): + # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions + assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + # Same with minconf=0 + assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) + assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) + # getunconfirmedbalance + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + + test_balances(fee_node_1=Decimal('0.01')) # Node 1 bumps the transaction fee and resends self.nodes[1].sendrawtransaction(txs[1]['hex']) + self.nodes[0].sendrawtransaction(txs[1]['hex']) # sending on both nodes is faster than waiting for propagation self.sync_all() self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + test_balances(fee_node_1=Decimal('0.02')) - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) # output of node 1's send - assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) - - self.nodes[1].generatetoaddress(1, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) self.sync_all() # balances are correct after the transactions are confirmed @@ -118,7 +131,7 @@ class WalletTest(BitcoinTestFramework): # Send total balance away from node 1 txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) self.nodes[1].sendrawtransaction(txs[0]['hex']) - self.nodes[1].generatetoaddress(2, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(2, ADDRESS_WATCHONLY) self.sync_all() # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago @@ -129,5 +142,62 @@ class WalletTest(BitcoinTestFramework): # getbalance with minconf=2 will show the new balance. assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0')) + # check mempool transactions count for wallet unconfirmed balance after + # dynamically loading the wallet. + before = self.nodes[1].getunconfirmedbalance() + dst = self.nodes[1].getnewaddress() + self.nodes[1].unloadwallet('') + self.nodes[0].sendtoaddress(dst, 0.1) + self.sync_all() + self.nodes[1].loadwallet('') + after = self.nodes[1].getunconfirmedbalance() + assert_equal(before + Decimal('0.1'), after) + + # Create 3 more wallet txs, where the last is not accepted to the + # mempool because it is the third descendant of the tx above + for _ in range(3): + # Set amount high enough such that all coins are spent by each tx + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 99) + + self.log.info('Check that wallet txs not in the mempool are untrusted') + assert txid not in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].gettransaction(txid)['trusted'], False) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) + + self.log.info("Test replacement and reorg of non-mempool tx") + tx_orig = self.nodes[0].gettransaction(txid)['hex'] + # Increase fee by 1 coin + tx_replace = tx_orig.replace( + struct.pack("<q", 99 * 10**8).hex(), + struct.pack("<q", 98 * 10**8).hex(), + ) + tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex'] + # Total balance is given by the sum of outputs of the tx + total_amount = sum([o['value'] for o in self.nodes[0].decoderawtransaction(tx_replace)['vout']]) + self.sync_all() + self.nodes[1].sendrawtransaction(hexstring=tx_replace, maxfeerate=0) + + # Now confirm tx_replace + block_reorg = self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY)[0] + self.sync_all() + assert_equal(self.nodes[0].getbalance(minconf=0), total_amount) + + self.log.info('Put txs back into mempool of node 1 (not node 0)') + self.nodes[0].invalidateblock(block_reorg) + self.nodes[1].invalidateblock(block_reorg) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted + self.nodes[0].generatetoaddress(1, ADDRESS_WATCHONLY) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted + + # Now confirm tx_orig + self.restart_node(1, ['-persistmempool=0']) + connect_nodes_bi(self.nodes, 0, 1) + sync_blocks(self.nodes) + self.nodes[1].sendrawtransaction(tx_orig) + self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) + self.sync_all() + assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin + + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 0477164b6b..daa834b5b8 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -13,8 +13,6 @@ from test_framework.util import ( assert_fee_amount, assert_raises_rpc_error, connect_nodes_bi, - sync_blocks, - sync_mempools, wait_until, ) @@ -34,7 +32,7 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size): """Return curr_balance after asserting the fee was in range""" @@ -59,9 +57,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(walletinfo['immature_balance'], 50) assert_equal(walletinfo['balance'], 0) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) self.nodes[1].generate(101) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) @@ -108,7 +106,7 @@ class WalletTest(BitcoinTestFramework): # Have node0 mine a block, thus it will collect its own fee. self.nodes[0].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] @@ -144,7 +142,7 @@ class WalletTest(BitcoinTestFramework): # Have node1 generate 100 blocks (so node0 can recover the fee) self.nodes[1].generate(100) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # node0 should end up with 100 btc in block rewards plus fees, but # minus the 21 plus fees sent to node2 @@ -168,12 +166,12 @@ class WalletTest(BitcoinTestFramework): txns_to_send.append(self.nodes[0].signrawtransactionwithwallet(raw_tx)) # Have node 1 (miner) send the transactions - self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], True) - self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], True) + self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], 0) + self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], 0) # Have node1 mine a block to confirm transactions: self.nodes[1].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 94) @@ -188,14 +186,14 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].settxfee(fee_per_byte * 1000) txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount txid = self.nodes[2].sendtoaddress(address, 10, "", "", True) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -203,7 +201,7 @@ class WalletTest(BitcoinTestFramework): # Sendmany 10 BTC txid = self.nodes[2].sendmany('', {address: 10}, 0, "", []) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_0_bal += Decimal('10') node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), node_0_bal) @@ -211,28 +209,14 @@ class WalletTest(BitcoinTestFramework): # Sendmany 10 BTC with subtract fee from amount txid = self.nodes[2].sendmany('', {address: 10}, 0, "", [address]) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) - # Test ResendWalletTransactions: - # Create a couple of transactions, then start up a fourth - # node (nodes[3]) and ask nodes[0] to rebroadcast. - # EXPECT: nodes[3] should have those transactions in its mempool. - txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - sync_mempools(self.nodes[0:2]) - self.start_node(3) connect_nodes_bi(self.nodes, 0, 3) - sync_blocks(self.nodes) - - relayed = self.nodes[0].resendwallettransactions() - assert_equal(set(relayed), {txid1, txid2}) - sync_mempools(self.nodes) - - assert txid1 in self.nodes[3].getrawmempool() + self.sync_all() # check if we can list zero value tx as available coins # 1. create raw_tx @@ -269,18 +253,18 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) txid_not_broadcast = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) self.nodes[1].generate(1) # mine a block, tx should not be in there - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[2].getbalance(), node_2_bal) # should not be changed because tx was not broadcasted # now broadcast from another node, mine a block, sync, and check the balance self.nodes[1].sendrawtransaction(tx_obj_not_broadcast['hex']) self.nodes[1].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal += 2 tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) assert_equal(self.nodes[2].getbalance(), node_2_bal) @@ -296,10 +280,10 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - sync_blocks(self.nodes[0:3]) + self.sync_blocks(self.nodes[0:3]) self.nodes[0].generate(1) - sync_blocks(self.nodes[0:3]) + self.sync_blocks(self.nodes[0:3]) node_2_bal += 2 # tx should be added to balance because after restarting the nodes tx should be broadcast @@ -356,7 +340,7 @@ class WalletTest(BitcoinTestFramework): address_to_import = self.nodes[2].getnewaddress() txid = self.nodes[0].sendtoaddress(address_to_import, 1) self.nodes[0].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # 2. Import address from node2 to node1 self.nodes[1].importaddress(address_to_import) @@ -382,15 +366,15 @@ class WalletTest(BitcoinTestFramework): coinbase_addr = self.nodes[1].getnewaddress() block_hash = self.nodes[0].generatetoaddress(1, coinbase_addr)[0] coinbase_txid = self.nodes[0].getblock(block_hash)['tx'][0] - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # Check that the txid and balance is found by node1 self.nodes[1].gettransaction(coinbase_txid) # check if wallet or blockchain maintenance changes the balance - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) blocks = self.nodes[0].generate(2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) balance_nodes = [self.nodes[i].getbalance() for i in range(3)] block_count = self.nodes[0].getblockcount() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 7d3d9b61e2..568b1f28d8 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -19,7 +19,13 @@ import io from test_framework.blocktools import add_witness_commitment, create_block, create_coinbase, send_to_witness from test_framework.messages import BIP125_SEQUENCE_NUMBER, CTransaction from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes, sync_mempools +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + connect_nodes_bi, + hex_str_to_bytes, +) WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -60,7 +66,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address) + test_simple_bumpfee_succeeds(self, rbf_node, peer_node, dest_address) test_segwit_bumpfee_succeeds(rbf_node, dest_address) test_nonrbf_bumpfee_fails(peer_node, dest_address) test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address) @@ -73,19 +79,23 @@ class BumpFeeTest(BitcoinTestFramework): test_unconfirmed_not_spendable(rbf_node, rbf_node_address) test_bumpfee_metadata(rbf_node, dest_address) test_locked_wallet_fails(rbf_node, dest_address) + test_change_script_match(rbf_node, dest_address) + # These tests wipe out a number of utxos that are expected in other tests + test_small_output_with_feerate_succeeds(rbf_node, dest_address) + test_no_more_inputs_fails(rbf_node, dest_address) self.log.info("Success") -def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): +def test_simple_bumpfee_succeeds(self, rbf_node, peer_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) rbftx = rbf_node.gettransaction(rbfid) - sync_mempools((rbf_node, peer_node)) + self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 # check that bumped_tx propagates, original tx was evicted and has a wallet conflict - sync_mempools((rbf_node, peer_node)) + self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert bumped_tx["txid"] in peer_node.getrawmempool() assert rbfid not in rbf_node.getrawmempool() @@ -173,6 +183,40 @@ def test_small_output_fails(rbf_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) assert_raises_rpc_error(-4, "Change output is too small", rbf_node.bumpfee, rbfid, {"totalFee": 50001}) +def test_small_output_with_feerate_succeeds(rbf_node, dest_address): + + # Make sure additional inputs exist + rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) + rbfid = spend_one_input(rbf_node, dest_address) + original_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_equal(len(original_input_list), 1) + original_txin = original_input_list[0] + # Keep bumping until we out-spend change output + tx_fee = 0 + while tx_fee < Decimal("0.0005"): + new_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + new_item = list(new_input_list)[0] + assert_equal(len(original_input_list), 1) + assert_equal(original_txin["txid"], new_item["txid"]) + assert_equal(original_txin["vout"], new_item["vout"]) + rbfid_new_details = rbf_node.bumpfee(rbfid) + rbfid_new = rbfid_new_details["txid"] + raw_pool = rbf_node.getrawmempool() + assert rbfid not in raw_pool + assert rbfid_new in raw_pool + rbfid = rbfid_new + tx_fee = rbfid_new_details["origfee"] + + # input(s) have been added + final_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_greater_than(len(final_input_list), 1) + # Original input is in final set + assert [txin for txin in final_input_list + if txin["txid"] == original_txin["txid"] + and txin["vout"] == original_txin["vout"]] + + rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) + assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) def test_dust_to_fee(rbf_node, dest_address): # check that if output is reduced to dust, it will be converted to fee @@ -272,19 +316,37 @@ def test_locked_wallet_fails(rbf_node, dest_address): rbf_node.walletlock() assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + +def test_change_script_match(rbf_node, dest_address): + """Test that the same change addresses is used for the replacement transaction when possible.""" + def get_change_address(tx): + tx_details = rbf_node.getrawtransaction(tx, 1) + txout_addresses = [txout['scriptPubKey']['addresses'][0] for txout in tx_details["vout"]] + return [address for address in txout_addresses if rbf_node.getaddressinfo(address)["ischange"]] + # Check that there is only one change output + rbfid = spend_one_input(rbf_node, dest_address) + change_addresses = get_change_address(rbfid) + assert_equal(len(change_addresses), 1) + + # Now find that address in each subsequent tx, and no other change + bumped_total_tx = rbf_node.bumpfee(rbfid, {"totalFee": 2000}) + assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'])) + bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) + assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) -def spend_one_input(node, dest_address): +def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): tx_input = dict( sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) - rawtx = node.createrawtransaction( - [tx_input], {dest_address: Decimal("0.00050000"), - node.getrawchangeaddress(): Decimal("0.00049000")}) + destinations = {dest_address: Decimal("0.00050000")} + if change_size > 0: + destinations[node.getrawchangeaddress()] = change_size + rawtx = node.createrawtransaction([tx_input], destinations) signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx["hex"]) return txid - def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -298,9 +360,15 @@ def submit_block_with_tx(node, tx): block.hashMerkleRoot = block.calc_merkle_root() add_witness_commitment(block) block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) return block +def test_no_more_inputs_fails(rbf_node, dest_address): + # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient + rbf_node.generatetoaddress(1, dest_address) + # spend all funds, no change output + rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) + assert_raises_rpc_error(-4, "Unable to create transaction: Insufficient funds", rbf_node.bumpfee, rbfid) if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 7b749235e2..0b584a0bb2 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 7ec72b8649..8ec4b98b6e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet arguments. @@ -40,8 +40,8 @@ class CreateWalletTest(BitcoinTestFramework): privkey = w0.dumpprivkey(addr) assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) - assert(not result[0]['success']) - assert('warning' not in result[0]) + assert not result[0]['success'] + assert 'warning' not in result[0] assert_equal(result[0]['error']['code'], -4) assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 6530c58c78..7c2ec56b5a 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. @@ -21,9 +21,9 @@ class DisableWalletTest (BitcoinTestFramework): # Make sure wallet is really disabled assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') - assert(x['isvalid'] == False) + assert x['isvalid'] == False x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert(x['isvalid'] == True) + assert x['isvalid'] == True # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 3f39654bb8..53edf710b9 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" @@ -46,10 +46,10 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): keypath = None if keytype == "inactivehdseed=1": # ensure the old master is still available - assert (hd_master_addr_old == addr) + assert hd_master_addr_old == addr elif keytype == "hdseed=1": # ensure we have generated a new hd master key - assert (hd_master_addr_old != addr) + assert hd_master_addr_old != addr hd_master_addr_ret = addr elif keytype == "script=1": # scripts don't have keypaths diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 61245e5104..97172d8b82 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 46462a16f3..9de30d0374 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet import RPCs. @@ -20,7 +20,12 @@ happened previously. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_raises_rpc_error, connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import ( + assert_raises_rpc_error, + connect_nodes, + assert_equal, + set_node_times, +) import collections import enum @@ -161,11 +166,12 @@ class ImportRescanTest(BitcoinTestFramework): timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] set_node_times(self.nodes, timestamp + TIMESTAMP_WINDOW + 1) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_all() # For each variation of wallet key import, invoke the import RPC and # check the results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run import for variant {}'.format(variant)) variant.expect_disabled = variant.rescan == Rescan.yes and variant.prune and variant.call == Call.single expect_rescan = variant.rescan == Rescan.yes and not variant.expect_disabled variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] @@ -187,10 +193,11 @@ class ImportRescanTest(BitcoinTestFramework): # Generate a block containing the new transactions. self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) - sync_blocks(self.nodes) + self.sync_all() # Check the latest results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run check for variant {}'.format(variant)) if not variant.expect_disabled: variant.expected_balance += variant.sent_amount variant.expected_txs += 1 diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index c6d0425f22..81c650f4c1 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC. @@ -25,7 +25,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - bytes_to_hex_str, ) from test_framework.wallet_util import ( get_key, @@ -127,7 +126,7 @@ class ImportMultiTest(BitcoinTestFramework): # Nonstandard scriptPubKey + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal flag") - nonstandardScriptPubKey = key.p2pkh_script + bytes_to_hex_str(CScript([OP_NOP])) + nonstandardScriptPubKey = key.p2pkh_script + CScript([OP_NOP]).hex() key = get_key(self.nodes[0]) self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, "timestamp": "now"}, @@ -630,7 +629,8 @@ class ImportMultiTest(BitcoinTestFramework): self.log.info("Should import a 1-of-2 bare multisig from descriptor") self.test_importmulti({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), "timestamp": "now"}, - success=True) + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") test_address(self.nodes[1], key1.p2pkh_addr, @@ -760,7 +760,7 @@ class ImportMultiTest(BitcoinTestFramework): assert_equal(addr2, newaddr2) # Import a multisig and make sure the keys don't go into the keypool - self.log.info('Imported scripts with pubkeys shoud not have their pubkeys go into the keypool') + self.log.info('Imported scripts with pubkeys should not have their pubkeys go into the keypool') addr1 = self.nodes[0].getnewaddress() addr2 = self.nodes[0].getnewaddress() pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 1116196268..e3aeb61197 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" @@ -21,7 +21,7 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') @@ -29,8 +29,8 @@ class KeyPoolTest(BitcoinTestFramework): addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid']) - assert(addr_data['hdseedid'] == wallet_info['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -59,7 +59,7 @@ class KeyPoolTest(BitcoinTestFramework): addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) - assert(len(addr) == 6) + assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index b7c8d3098d..0014555ade 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. @@ -17,7 +17,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, - sync_blocks, ) @@ -55,11 +54,11 @@ class KeypoolRestoreTest(BitcoinTestFramework): # Make sure we're creating the outputs we expect address_details = self.nodes[idx].validateaddress(addr_extpool) if i == 0: - assert(not address_details["isscript"] and not address_details["iswitness"]) + assert not address_details["isscript"] and not address_details["iswitness"] elif i == 1: - assert(address_details["isscript"] and not address_details["iswitness"]) + assert address_details["isscript"] and not address_details["iswitness"] else: - assert(not address_details["isscript"] and address_details["iswitness"]) + assert not address_details["isscript"] and address_details["iswitness"] self.log.info("Send funds to wallet") @@ -67,7 +66,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.nodes[0].generate(1) self.nodes[0].sendtoaddress(addr_extpool, 5) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.log.info("Restart node with wallet backup") self.stop_node(idx) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 011975e371..5e94068930 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" @@ -10,7 +10,6 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_raises_rpc_error, - sync_blocks, ) @@ -24,7 +23,7 @@ class ReceivedByTest(BitcoinTestFramework): def run_test(self): # Generate block to get out of IBD self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 25ab222375..021a29d4ac 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listsincelast RPC.""" @@ -98,7 +98,8 @@ class ListSinceBlockTest (BitcoinTestFramework): self.nodes[2].generate(7) self.log.info('lastblockhash=%s' % (lastblockhash)) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) self.join_network() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 17f044bf65..997d6e702c 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" @@ -11,9 +11,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, assert_equal, - bytes_to_hex_str, hex_str_to_bytes, - sync_mempools, ) def tx_from_hex(hexstring): @@ -126,9 +124,9 @@ class ListTransactionsTest(BitcoinTestFramework): # 1. Chain a few transactions that don't opt-in. txid_1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - assert(not is_opt_in(self.nodes[0], txid_1)) + assert not is_opt_in(self.nodes[0], txid_1) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) # Tx2 will build off txid_1, still not opting in to RBF. @@ -146,9 +144,9 @@ class ListTransactionsTest(BitcoinTestFramework): txid_2 = self.nodes[1].sendrawtransaction(tx2_signed) # ...and check the result - assert(not is_opt_in(self.nodes[1], txid_2)) + assert not is_opt_in(self.nodes[1], txid_2) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) # Tx3 will opt-in to RBF @@ -158,13 +156,13 @@ class ListTransactionsTest(BitcoinTestFramework): tx3 = self.nodes[0].createrawtransaction(inputs, outputs) tx3_modified = tx_from_hex(tx3) tx3_modified.vin[0].nSequence = 0 - tx3 = bytes_to_hex_str(tx3_modified.serialize()) + tx3 = tx3_modified.serialize().hex() tx3_signed = self.nodes[0].signrawtransactionwithwallet(tx3)['hex'] txid_3 = self.nodes[0].sendrawtransaction(tx3_signed) - assert(is_opt_in(self.nodes[0], txid_3)) + assert is_opt_in(self.nodes[0], txid_3) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) # Tx4 will chain off tx3. Doesn't signal itself, but depends on one @@ -176,21 +174,21 @@ class ListTransactionsTest(BitcoinTestFramework): tx4_signed = self.nodes[1].signrawtransactionwithwallet(tx4)["hex"] txid_4 = self.nodes[1].sendrawtransaction(tx4_signed) - assert(not is_opt_in(self.nodes[1], txid_4)) + assert not is_opt_in(self.nodes[1], txid_4) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee - tx3_b = bytes_to_hex_str(tx3_b.serialize()) + tx3_b = tx3_b.serialize().hex() tx3_b_signed = self.nodes[0].signrawtransactionwithwallet(tx3_b)['hex'] - txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, True) - assert(is_opt_in(self.nodes[0], txid_3b)) + txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, 0) + assert is_opt_in(self.nodes[0], txid_3b) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) # Check gettransaction as well: @@ -203,7 +201,7 @@ class ListTransactionsTest(BitcoinTestFramework): # After mining a transaction, it's no longer BIP125-replaceable self.nodes[0].generate(1) - assert(txid_3b not in self.nodes[0].getrawmempool()) + assert txid_3b not in self.nodes[0].getrawmempool() assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index df778f57df..984ffab5a4 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 00bf58d709..5810e94938 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -1,32 +1,77 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test resendwallettransactions RPC.""" +"""Test that the wallet resends transactions periodically.""" +from collections import defaultdict +import time +from test_framework.blocktools import create_block, create_coinbase +from test_framework.messages import ToHex +from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, wait_until + +class P2PStoreTxInvs(P2PInterface): + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == 1: + # save txid + self.tx_invs_received[i.hash] += 1 class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['--walletbroadcast=false']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): - # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + node = self.nodes[0] # alias + + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a new transaction and wait until it's broadcast") + txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + + # Wallet rebroadcast is first scheduled 1 sec after startup (see + # nNextResend in ResendWalletTransactions()). Sleep for just over a + # second to be certain that it has been called before the first + # setmocktime call below. + time.sleep(1.1) + + # Can take a few seconds due to transaction trickling + wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + + # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a block") + # Create and submit a block without the transaction. + # Transactions are only rebroadcast if there has been a block at least five minutes + # after the last time we tried to broadcast. Use mocktime and give an extra minute to be sure. + block_time = int(time.time()) + 6 * 60 + node.setmocktime(block_time) + block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockchaininfo()['blocks']), block_time) + block.nVersion = 3 + block.rehash() + block.solve() + node.submitblock(ToHex(block)) - # Should return an empty array if there aren't unconfirmed wallet transactions. - self.stop_node(0) - self.start_node(0, extra_args=[]) - assert_equal(self.nodes[0].resendwallettransactions(), []) + # Transaction should not be rebroadcast + node.p2ps[1].sync_with_ping() + assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - # Should return an array with the unconfirmed wallet transaction. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - assert_equal(self.nodes[0].resendwallettransactions(), [txid]) + self.log.info("Transaction should be rebroadcast after 30 minutes") + # Use mocktime and give an extra 5 minutes to be sure. + rebroadcast_time = int(time.time()) + 41 * 60 + node.setmocktime(rebroadcast_time) + wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) if __name__ == '__main__': ResendWalletTransactionsTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index a0a35bf43d..60d7205887 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" @@ -8,10 +8,8 @@ import io from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str as b2x, connect_nodes, disconnect_nodes, - sync_blocks, ) from test_framework.messages import CTransaction, COIN @@ -82,13 +80,13 @@ class TxnMallTest(BitcoinTestFramework): # Use a different signature hash type to sign. This creates an equivalent but malleated clone. # Don't send the clone anywhere yet - tx1_clone = self.nodes[0].signrawtransactionwithwallet(b2x(clone_tx.serialize()), None, "ALL|ANYONECANPAY") + tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_tx.serialize().hex(), None, "ALL|ANYONECANPAY") assert_equal(tx1_clone["complete"], True) # Have node0 mine a block, if requested: if (self.options.mine_block): self.nodes[0].generate(1) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -124,7 +122,7 @@ class TxnMallTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(node0_tx2["hex"]) self.nodes[2].sendrawtransaction(tx2["hex"]) self.nodes[2].generate(1) # Mine another block to make sure we sync - sync_blocks(self.nodes) + self.sync_blocks() # Re-fetch transaction info: tx1 = self.nodes[0].gettransaction(txid1) diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index 9050e21247..40eeb4048c 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" @@ -11,7 +11,6 @@ from test_framework.util import ( connect_nodes, disconnect_nodes, find_output, - sync_blocks, ) class TxnMallTest(BitcoinTestFramework): @@ -86,7 +85,7 @@ class TxnMallTest(BitcoinTestFramework): # Have node0 mine a block: if (self.options.mine_block): self.nodes[0].generate(1) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -119,7 +118,7 @@ class TxnMallTest(BitcoinTestFramework): # Reconnect the split network, and sync chain: connect_nodes(self.nodes[1], 2) self.nodes[2].generate(1) # Mine another block to make sure we sync - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].gettransaction(doublespend_txid)["confirmations"], 2) # Re-fetch transaction info: diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index c370ce0c04..1d6122a13d 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,26 +12,23 @@ Author: @MarcoFalke from subprocess import check_output import re -import sys FOLDER_GREP = 'src' FOLDER_TEST = 'src/test/' REGEX_ARG = '(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"' REGEX_DOC = 'AddArg\("(-[^"=]+?)(?:=|")' -CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/{}'.format(FOLDER_GREP) +CMD_ROOT_DIR = '$(git rev-parse --show-toplevel)/{}'.format(FOLDER_GREP) CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST) +CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWalletOptions' -- {} | grep AddArg".format(CMD_ROOT_DIR) +CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) -def main(): - if sys.version_info >= (3, 6): - used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') - docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') - else: - used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() - docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() +def lint_missing_argument_documentation(): + used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() + docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) @@ -45,7 +42,24 @@ def main(): print("Args unknown : {}".format(len(args_unknown))) print(args_unknown) - sys.exit(len(args_need_doc)) + assert 0 == len(args_need_doc), "Please document the following arguments: {}".format(args_need_doc) + + +def lint_missing_hidden_wallet_args(): + wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True).decode('utf8').strip() + wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True).decode('utf8').strip() + + wallet_args = set(re.findall(re.compile(REGEX_DOC), wallet_args)) + wallet_hidden_args = set(re.findall(re.compile(r' "([^"=]+)'), wallet_hidden_args)) + + hidden_missing = wallet_args.difference(wallet_hidden_args) + if hidden_missing: + assert 0, "Please add {} to the hidden args in DummyWalletInit::AddWalletOptions".format(hidden_missing) + + +def main(): + lint_missing_argument_documentation() + lint_missing_hidden_wallet_args() if __name__ == "__main__": diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index be67cbed31..e1a99abc49 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -10,10 +10,9 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" - "checkpoints -> validation -> checkpoints" "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" - "policy/policy -> validation -> policy/policy" + "policy/policy -> policy/settings -> policy/policy" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bantablemodel -> qt/clientmodel -> qt/bantablemodel" "qt/bitcoingui -> qt/utilitydialog -> qt/bitcoingui" @@ -30,14 +29,13 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "wallet/coincontrol -> wallet/wallet -> wallet/coincontrol" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "policy/fees -> policy/policy -> validation -> policy/fees" + "policy/fees -> txmempool -> validation -> policy/fees" "policy/rbf -> txmempool -> validation -> policy/rbf" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/addressbookpage" "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/guiutil" "txmempool -> validation -> validationinterface -> txmempool" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/receivecoinsdialog -> qt/addressbookpage" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/signverifymessagedialog -> qt/addressbookpage" - "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/intro -> qt/guiutil" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/sendcoinsdialog -> qt/sendcoinsentry -> qt/addressbookpage" ) diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index f5d4780b6d..224e62f04a 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -39,7 +39,7 @@ def parse_function_calls(function_name, source_code): >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ - assert(type(function_name) is str and type(source_code) is str and function_name) + assert type(function_name) is str and type(source_code) is str and function_name lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] @@ -53,7 +53,7 @@ def normalize(s): >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ - assert(type(s) is str) + assert type(s) is str s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub("/\*.*?\*/", " ", s) @@ -77,7 +77,7 @@ def escape(s): >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s @@ -92,7 +92,7 @@ def unescape(s): >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s @@ -151,10 +151,10 @@ def parse_function_call_and_arguments(function_name, function_call): >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] """ - assert(type(function_name) is str and type(function_call) is str and function_name) + assert type(function_name) is str and type(function_call) is str and function_name remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) - assert(remaining.startswith(expected_function_call)) + assert remaining.startswith(expected_function_call) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 @@ -213,7 +213,7 @@ def parse_string_content(argument): >>> parse_string_content('1 2 3') '' """ - assert(type(argument) is str) + assert type(argument) is str string_content = "" in_string = False for char in normalize(escape(argument)): @@ -240,7 +240,7 @@ def count_format_specifiers(format_string): >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ - assert(type(format_string) is str) + assert type(format_string) is str format_string = format_string.replace('%%', 'X') n = 0 in_specifier = False @@ -262,27 +262,27 @@ def main(): parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " "argument (e.g. 1 in the case of fprintf)", default=0) parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", type=argparse.FileType("r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args() - exit_code = 0 - for f in args.file: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue sys.exit(exit_code) diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh index 863caa9d5c..588ba428d7 100755 --- a/test/lint/lint-python-dead-code.sh +++ b/test/lint/lint-python-dead-code.sh @@ -15,5 +15,5 @@ fi vulture \ --min-confidence 60 \ - --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,get_ecdh_key,get_privkey,is_compressed,is_fullyvalid,msg_generic,on_*,optionxform,restype,set_privkey,profile_with_perf" \ + --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,is_compressed,is_valid,verify_ecdsa,msg_generic,on_*,optionxform,restype,profile_with_perf" \ $(git ls-files -- "*.py" ":(exclude)contrib/" ":(exclude)test/functional/data/invalid_txs.py") diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index d55119b266..643272de52 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -15,6 +15,7 @@ unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/chacha20.cpp unsigned-integer-overflow:crypto/ctaes/ctaes.c +unsigned-integer-overflow:crypto/poly1305.cpp unsigned-integer-overflow:crypto/ripemd160.cpp unsigned-integer-overflow:crypto/sha1.cpp unsigned-integer-overflow:crypto/sha256.cpp |