diff options
257 files changed, 4719 insertions, 3911 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index cb6a920060..ac39fe235b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,15 +11,15 @@ environment: PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 cache: -- C:\tools\vcpkg\installed -> appveyor.yml -- C:\Users\appveyor\clcache -> appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in +- C:\tools\vcpkg\installed -> .appveyor.yml +- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq +- cmd: echo set(VCPKG_BUILD_TYPE release) >> C:\tools\vcpkg\triplets\%PLATFORM%-windows-static.cmake - cmd: vcpkg remove --outdated --recurse - cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL -- cmd: del /s /q C:\Tools\vcpkg\installed\%PLATFORM%-windows-static\debug # Remove unused debug library before_build: - ps: clcache -M 536870912 - cmd: python build_msvc\msvc-autogen.py diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..9104a0a3d1 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,26 @@ +task: + name: "FreeBsd 12.0 amd64 [GOAL: install] [no depends, only system libs]" + freebsd_instance: + image: freebsd-12-0-release-amd64 + ccache_cache: + folder: "/tmp/ccache_dir" + env: + MAKEJOBS: "-j3" + CONFIGURE_OPTS: "--disable-dependency-tracking" + GOAL: "install" + CCACHE_SIZE: "200M" + CCACHE_COMPRESS: 1 + CCACHE_DIR: "/tmp/ccache_dir" + install_script: + - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache + - ./contrib/install_db4.sh $(pwd) + - ccache --max-size=${CCACHE_SIZE} + configure_script: + - ./autogen.sh + - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false) + make_script: + - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false ) + check_script: + - gmake check ${MAKEJOBS} VERBOSE=1 + functional_test_script: + - ./test/functional/test_runner.py --ci --combinedlogslen=1000 --quiet --failfast diff --git a/.python-version b/.python-version index 7bcbb3808b..c49282585a 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.4.9 +3.5.6 diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000..69d8c6aee4 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,261 @@ +[style] +# Align closing bracket with visual indentation. +align_closing_bracket_with_visual_indent=True + +# Allow dictionary keys to exist on multiple lines. For example: +# +# x = { +# ('this is the first element of a tuple', +# 'this is the second element of a tuple'): +# value, +# } +allow_multiline_dictionary_keys=False + +# Allow lambdas to be formatted on more than one line. +allow_multiline_lambdas=False + +# Allow splits before the dictionary value. +allow_split_before_dict_value=True + +# Number of blank lines surrounding top-level function and class +# definitions. +blank_lines_around_top_level_definition=2 + +# Insert a blank line before a class-level docstring. +blank_line_before_class_docstring=False + +# Insert a blank line before a module docstring. +blank_line_before_module_docstring=False + +# Insert a blank line before a 'def' or 'class' immediately nested +# within another 'def' or 'class'. For example: +# +# class Foo: +# # <------ this blank line +# def method(): +# ... +blank_line_before_nested_class_or_def=False + +# Do not split consecutive brackets. Only relevant when +# dedent_closing_brackets is set. For example: +# +# call_func_that_takes_a_dict( +# { +# 'key1': 'value1', +# 'key2': 'value2', +# } +# ) +# +# would reformat to: +# +# call_func_that_takes_a_dict({ +# 'key1': 'value1', +# 'key2': 'value2', +# }) +coalesce_brackets=False + +# The column limit. +column_limit=160 + +# The style for continuation alignment. Possible values are: +# +# - SPACE: Use spaces for continuation alignment. This is default behavior. +# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns +# (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation +# alignment. +# - LESS: Slightly left if cannot vertically align continuation lines with +# indent characters. +# - VALIGN-RIGHT: Vertically align continuation lines with indent +# characters. Slightly right (one more indent character) if cannot +# vertically align continuation lines with indent characters. +# +# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is +# enabled. +continuation_align_style=SPACE + +# Indent width used for line continuations. +continuation_indent_width=4 + +# Put closing brackets on a separate line, dedented, if the bracketed +# expression can't fit in a single line. Applies to all kinds of brackets, +# including function definitions and calls. For example: +# +# config = { +# 'key1': 'value1', +# 'key2': 'value2', +# } # <--- this bracket is dedented and on a separate line +# +# time_series = self.remote_client.query_entity_counters( +# entity='dev3246.region1', +# key='dns.query_latency_tcp', +# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), +# start_ts=now()-timedelta(days=3), +# end_ts=now(), +# ) # <--- this bracket is dedented and on a separate line +dedent_closing_brackets=False + +# Disable the heuristic which places each list element on a separate line +# if the list is comma-terminated. +disable_ending_comma_heuristic=False + +# Place each dictionary entry onto its own line. +each_dict_entry_on_separate_line=True + +# The regex for an i18n comment. The presence of this comment stops +# reformatting of that line, because the comments are required to be +# next to the string they translate. +i18n_comment= + +# The i18n function call names. The presence of this function stops +# reformattting on that line, because the string it has cannot be moved +# away from the i18n comment. +i18n_function_call= + +# Indent the dictionary value if it cannot fit on the same line as the +# dictionary key. For example: +# +# config = { +# 'key1': +# 'value1', +# 'key2': value1 + +# value2, +# } +indent_dictionary_value=False + +# The number of columns to use for indentation. +indent_width=4 + +# Join short lines into one line. E.g., single line 'if' statements. +join_multiple_lines=True + +# Do not include spaces around selected binary operators. For example: +# +# 1 + 2 * 3 - 4 / 5 +# +# will be formatted as follows when configured with "*,/": +# +# 1 + 2*3 - 4/5 +# +no_spaces_around_selected_binary_operators= + +# Use spaces around default or named assigns. +spaces_around_default_or_named_assign=False + +# Use spaces around the power operator. +spaces_around_power_operator=False + +# The number of spaces required before a trailing comment. +spaces_before_comment=2 + +# Insert a space between the ending comma and closing bracket of a list, +# etc. +space_between_ending_comma_and_closing_bracket=True + +# Split before arguments +split_all_comma_separated_values=False + +# Split before arguments if the argument list is terminated by a +# comma. +split_arguments_when_comma_terminated=False + +# Set to True to prefer splitting before '&', '|' or '^' rather than +# after. +split_before_bitwise_operator=True + +# Split before the closing bracket if a list or dict literal doesn't fit on +# a single line. +split_before_closing_bracket=True + +# Split before a dictionary or set generator (comp_for). For example, note +# the split before the 'for': +# +# foo = { +# variable: 'Hello world, have a nice day!' +# for variable in bar if variable != 42 +# } +split_before_dict_set_generator=True + +# Split before the '.' if we need to split a longer expression: +# +# foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) +# +# would reformat to something like: +# +# foo = ('This is a really long string: {}, {}, {}, {}' +# .format(a, b, c, d)) +split_before_dot=False + +# Split after the opening paren which surrounds an expression if it doesn't +# fit on a single line. +split_before_expression_after_opening_paren=False + +# If an argument / parameter list is going to be split, then split before +# the first argument. +split_before_first_argument=False + +# Set to True to prefer splitting before 'and' or 'or' rather than +# after. +split_before_logical_operator=True + +# Split named assignments onto individual lines. +split_before_named_assigns=True + +# Set to True to split list comprehensions and generators that have +# non-trivial expressions and multiple clauses before each of these +# clauses. For example: +# +# result = [ +# a_long_var + 100 for a_long_var in xrange(1000) +# if a_long_var % 10] +# +# would reformat to something like: +# +# result = [ +# a_long_var + 100 +# for a_long_var in xrange(1000) +# if a_long_var % 10] +split_complex_comprehension=False + +# The penalty for splitting right after the opening bracket. +split_penalty_after_opening_bracket=30 + +# The penalty for splitting the line after a unary operator. +split_penalty_after_unary_operator=10000 + +# The penalty for splitting right before an if expression. +split_penalty_before_if_expr=0 + +# The penalty of splitting the line around the '&', '|', and '^' +# operators. +split_penalty_bitwise_operator=300 + +# The penalty for splitting a list comprehension or generator +# expression. +split_penalty_comprehension=80 + +# The penalty for characters over the column limit. +split_penalty_excess_character=7000 + +# The penalty incurred by adding a line split to the unwrapped line. The +# more line splits added the higher the penalty. +split_penalty_for_added_line_split=30 + +# The penalty of splitting a list of "import as" names. For example: +# +# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, +# long_argument_2, +# long_argument_3) +# +# would reformat to something like: +# +# from a_very_long_or_indented_module_name_yada_yad import ( +# long_argument_1, long_argument_2, long_argument_3) +split_penalty_import_names=0 + +# The penalty of splitting the line around the 'and' and 'or' +# operators. +split_penalty_logical_operator=300 + +# Use the Tab character for indentation. +use_tabs=False + diff --git a/.travis.yml b/.travis.yml index 0ede31b4a6..ef8da16936 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ jobs: env: cache: false language: python - python: '3.4' # Oldest supported version according to doc/dependencies.md + python: '3.5' # Oldest supported version according to doc/dependencies.md install: - set -o errexit; source .travis/lint_04_install.sh before_script: @@ -158,7 +158,7 @@ jobs: name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' env: >- HOST=x86_64-apple-darwin14 - PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" OSX_SDK=10.11 RUN_UNIT_TESTS=false RUN_FUNCTIONAL_TESTS=false diff --git a/.travis/lint_06_script.sh b/.travis/lint_06_script.sh index 701e6d8005..1a1693a8ce 100755 --- a/.travis/lint_06_script.sh +++ b/.travis/lint_06_script.sh @@ -21,5 +21,5 @@ test/lint/lint-all.sh if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys && - travis_wait 50 contrib/verify-commits/verify-commits.py --clean-merge=2; + ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 139fe7dc52..007ebd7ccf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,7 +62,7 @@ Commit messages should be verbose by default consisting of a short subject line paragraph(s), unless the title alone is self-explanatory (like "Corrected typo in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for -your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). +your decisions. Further explanation [here](https://chris.beams.io/posts/git-commit/). If a particular commit references another issue, please add the reference. For example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords @@ -239,7 +239,10 @@ consensus to merge a pull request (remember that discussions may have been spread out over GitHub, mailing list and IRC discussions). The following language is used within pull-request comments: - - ACK means "I have tested the code and I agree it should be merged"; + - (t)ACK means "I have tested the code and I agree it should be merged", involving + change-specific manual testing in addition to running the unit and functional + tests, and in case it is not obvious how the manual testing was done, it should + be described; - NACK means "I disagree this should be merged", and must be accompanied by sound technical justification (or in certain cases of copyright/patent/licensing issues, legal justification). NACKs without accompanying reasoning may be @@ -66,7 +66,7 @@ Translations ------------ Changes to translations as well as new translations can be submitted to -[Bitcoin Core's Transifex page](https://www.transifex.com/projects/p/bitcoin/). +[Bitcoin Core's Transifex page](https://www.transifex.com/bitcoin/bitcoin/). Translations are periodically pulled from Transifex and merged into the git repository. See the [translation process](doc/translation_process.md) for details on how this works. diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj index 771e8a56f4..0723243cd7 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj @@ -32,6 +32,7 @@ <ClCompile Include="..\..\src\bench\examples.cpp" /> <ClCompile Include="..\..\src\bench\lockedpool.cpp" /> <ClCompile Include="..\..\src\bench\mempool_eviction.cpp" /> + <ClCompile Include="..\..\src\bench\rpc_mempool.cpp" /> <ClCompile Include="..\..\src\bench\merkle_root.cpp" /> <ClCompile Include="..\..\src\bench\rollingbloom.cpp" /> <ClCompile Include="..\..\src\bench\verify_script.cpp" /> @@ -146,7 +147,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -164,7 +164,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -180,7 +179,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -198,7 +196,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index 6c82b1e7de..6b668054a3 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -113,7 +113,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -131,7 +130,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -147,7 +145,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -165,7 +162,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj index c52dfdb28c..1d0174e319 100644 --- a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj +++ b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj @@ -116,7 +116,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -134,7 +133,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -150,7 +148,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -168,7 +165,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index a83529c782..fa86da7bae 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -134,7 +134,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -152,7 +151,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -168,7 +166,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -186,7 +183,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 9a42f141c9..bb212af52e 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -79,7 +79,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -96,7 +95,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -115,7 +113,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -138,7 +135,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/common.vcxproj b/build_msvc/common.vcxproj index 93004dda7c..889dc6c2ad 100644 --- a/build_msvc/common.vcxproj +++ b/build_msvc/common.vcxproj @@ -24,6 +24,7 @@ <DisableSpecificWarnings>4018;4244;4267;4715;4805;</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> </ItemDefinitionGroup> </Project> diff --git a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in index 1b24acd8ce..5849e463a6 100644 --- a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in +++ b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in @@ -99,7 +99,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -116,7 +115,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -131,7 +129,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -148,7 +145,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index 459e81eb0c..292e193f5d 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -91,7 +91,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> @@ -109,7 +108,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> @@ -129,7 +127,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> @@ -151,7 +148,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> diff --git a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in index ad183d4904..e7002036ad 100644 --- a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in +++ b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in @@ -89,7 +89,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -104,7 +103,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -121,7 +119,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -140,7 +137,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in index acb827bd95..48a117bcfe 100644 --- a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in +++ b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in @@ -89,9 +89,8 @@ <PrecompiledHeader>NotUsing</PrecompiledHeader> <WarningLevel>Level3</WarningLevel> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -104,9 +103,8 @@ <PrecompiledHeader>NotUsing</PrecompiledHeader> <WarningLevel>Level3</WarningLevel> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -122,9 +120,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -141,9 +138,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index 855c9353fc..dbd91cf4db 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -92,7 +92,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -112,7 +111,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -134,7 +132,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -158,7 +155,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index b19f1e2396..33f4054546 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -131,7 +129,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -151,7 +148,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> diff --git a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in index 2e32c25762..187d955687 100644 --- a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in +++ b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -131,7 +129,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -151,7 +148,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> diff --git a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in index 7aba987cd1..c877a280c0 100644 --- a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in +++ b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -131,7 +129,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -151,7 +148,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj index 32ea070a05..a32aafbd74 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj @@ -126,7 +126,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> @@ -143,7 +142,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> @@ -162,7 +160,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -183,7 +180,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 03db97c647..a08e708f81 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -25,7 +25,7 @@ <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\test_bitcoin.cpp" /> - <ClCompile Include="..\..\src\test\test_bitcoin_main.cpp" /> + <ClCompile Include="..\..\src\test\main.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> </ItemGroup> <ItemGroup> @@ -135,7 +135,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -152,7 +151,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -170,7 +168,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -190,7 +187,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj index db2f8a6216..a2f5659049 100644 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ b/build_msvc/testconsensus/testconsensus.vcxproj @@ -95,7 +95,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -110,7 +109,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -129,7 +127,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -148,7 +145,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/configure.ac b/configure.ac index dec6d4fac3..a3ba8ce808 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 17) +define(_CLIENT_VERSION_MINOR, 18) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -85,8 +85,8 @@ AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.4 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.4 python3.5 python3.6 python3.7 python3 python]) +dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 6ee65f40be..0c8c396503 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -7,6 +7,8 @@ clang-format-diff.py A script to format unified git diffs according to [.clang-format](../../src/.clang-format). +Requires `clang-format`, installed e.g. via `brew install clang-format` on macOS. + For instance, to format the last commit with 0 lines of context, the script should be called from the git root folder as follows. @@ -167,7 +169,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage after a gitian build: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py If only supported symbols are used the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index abfa5ed5ae..2e4657f1dd 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -8,9 +8,18 @@ MAPPING = { 'core_write.cpp': 'core_io.cpp', } +# Directories with header-based modules, where the assumption that .cpp files +# define functions and variables declared in corresponding .h files is +# incorrect. +HEADER_MODULE_PATHS = [ + 'interfaces/' +] + def module_name(path): if path in MAPPING: path = MAPPING[path] + if any(path.startswith(dirpath) for dirpath in HEADER_MODULE_PATHS): + return path if path.endswith(".h"): return path[:-2] if path.endswith(".c"): diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index ab834b7623..41a640b464 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -47,13 +47,12 @@ def git_config_get(option, default=None): except subprocess.CalledProcessError: return default -def retrieve_pr_info(repo,pull,ghtoken): +def retrieve_json(req, ghtoken): ''' - Retrieve pull request information from github. - Return None if no title can be found, or an error happens. + Retrieve json from github. + Return None if an error happens. ''' try: - req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) if ghtoken is not None: req.add_header('Authorization', 'token ' + ghtoken) result = urlopen(req) @@ -69,6 +68,18 @@ def retrieve_pr_info(repo,pull,ghtoken): print('Warning: unable to retrieve pull information from github: %s' % e) return None +def retrieve_pr_info(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + return retrieve_json(req,ghtoken) + +def retrieve_pr_comments(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments") + return retrieve_json(req,ghtoken) + +def retrieve_pr_reviews(repo,pull,ghtoken): + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews") + return retrieve_json(req,ghtoken) + def ask_prompt(text): print(text,end=" ",file=stderr) stderr.flush() @@ -133,6 +144,16 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def get_acks_from_comments(head_commit, comments): + assert len(head_commit) == 6 + ack_str ='\n\nACKs for commit {}:\n'.format(head_commit) + for c in comments: + review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_commit in l] + if review: + ack_str += ' {}:\n'.format(c['user']['login']) + ack_str += ' {}\n'.format(review[0]) + return ack_str + def print_merge_details(pull, title, branch, base_branch, head_branch): print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) @@ -185,6 +206,9 @@ def main(): info = retrieve_pr_info(repo,pull,ghtoken) if info is None: sys.exit(1) + comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) + if comments is None: + sys.exit(1) title = info['title'].strip() body = info['body'].strip() # precedence order for destination branch argument: @@ -238,6 +262,7 @@ def main(): message = firstline + '\n\n' message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' + message += get_acks_from_comments(head_commit=subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8')[:6], comments=comments) try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) except subprocess.CalledProcessError: diff --git a/contrib/devtools/split-debug.sh.in b/contrib/devtools/split-debug.sh.in index deda49cc54..92b72b1446 100644 --- a/contrib/devtools/split-debug.sh.in +++ b/contrib/devtools/split-debug.sh.in @@ -1,5 +1,5 @@ #!/bin/sh - +set -e if [ $# -ne 3 ]; then echo "usage: $0 <input> <stripped-binary> <debug-binary>" fi diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c6158c9422..7729dd7257 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -9,7 +9,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import re diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000..16d03e1fff --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) + "denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256()); + "fs_tests/fsbridge_fstream" # deterministic test failure? + "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() + "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() + "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) +) + +TEST_BITCOIN_BINARY="src/test/test_bitcoin" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e97072c80a..5845d8fd89 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-linux-0.18" +name: "bitcoin-core-linux-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -30,12 +31,13 @@ packages: - "faketime" - "bsdmainutils" - "ca-certificates" -- "python" +- "python3" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" @@ -179,8 +181,8 @@ script: | find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg cp ../doc/README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 297a136fae..4cfca403b1 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,5 +1,6 @@ --- name: "bitcoin-dmg-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -12,6 +13,8 @@ remotes: files: - "bitcoin-osx-unsigned.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH=`pwd`:$PATH diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 87d8007ccb..24292d089a 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-osx-0.18" +name: "bitcoin-core-osx-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -23,9 +24,9 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" +- "python3" +- "python3-dev" +- "python3-setuptools" - "fonts-tuffy" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" @@ -33,6 +34,8 @@ remotes: files: - "MacOSX10.11.sdk.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin14" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 045be873e9..656c6d9b7a 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,5 +1,6 @@ --- name: "bitcoin-win-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -16,6 +17,8 @@ files: - "osslsigncode-Backports-to-1.7.1.patch" - "bitcoin-win-unsigned.tar.gz" script: | + set -e -o pipefail + BUILD_DIR=`pwd` SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 31b9c309c7..eca32a5dc5 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,6 +1,7 @@ --- -name: "bitcoin-win-0.18" +name: "bitcoin-core-win-0.19" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -20,13 +21,15 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" +- "python3" - "rename" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 80c18aa889..33f0f7e5b0 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -6,6 +6,7 @@ C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random +6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 2985106982..25a1c7351a 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,6 +1,5 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts -run using Python 3 but are compatible with Python 2. +Construct a linear, no-fork, best version of the Bitcoin blockchain. ## Step 1: Download hash list diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 56c1fbfc92..468aec04b5 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -16,7 +16,7 @@ import hashlib import datetime import time from collections import namedtuple -from binascii import hexlify, unhexlify +from binascii import unhexlify settings = {} @@ -61,7 +61,7 @@ def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') + hash_str = hash.hex() return hash_str def get_blk_dt(blk_hdr): @@ -213,7 +213,7 @@ class BlockDataCopier: inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) + print("Invalid magic: " + inMagic.hex()) return inLenLE = inhdr[4:] su = struct.unpack("<I", inLenLE) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index 27ca15acb4..1215962a16 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -37,7 +37,8 @@ Configuration files Import trusted keys ------------------- -In order to check the commit signatures you must add the trusted PGP keys to your machine. This can be done in Linux by running +In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: + ```sh gpg --recv-keys $(<contrib/verify-commits/trusted-keys) ``` diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 6bbed01073..255ce75092 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -5,6 +5,7 @@ """Verify commits against a trusted keys list.""" import argparse import hashlib +import logging import os import subprocess import sys @@ -66,6 +67,11 @@ def tree_sha512sum(commit='HEAD'): return overall.hexdigest() def main(): + + # Enable debug logging if running in CI + if 'CI' in os.environ and os.environ['CI'].lower() == "true": + logging.getLogger().setLevel(logging.DEBUG) + # Parse arguments parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') @@ -95,6 +101,10 @@ def main(): # Iterate through commits while True: + + # Log a message to prevent Travis from timing out + logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8])) + if current_commit == verified_root: print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) sys.exit(0) diff --git a/contrib/windeploy/win-codesign.cert b/contrib/windeploy/win-codesign.cert index 200b30a3f0..5bc5dc5809 100644 --- a/contrib/windeploy/win-codesign.cert +++ b/contrib/windeploy/win-codesign.cert @@ -1,99 +1,100 @@ -----BEGIN CERTIFICATE----- -MIIFTTCCBDWgAwIBAgIRALlW05RLwG2hMQMX5d/o5J8wDQYJKoZIhvcNAQELBQAw -fTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxIzAhBgNV -BAMTGkNPTU9ETyBSU0EgQ29kZSBTaWduaW5nIENBMB4XDTE2MDIwMzAwMDAwMFoX -DTE5MDMwNTIzNTk1OVowgbUxCzAJBgNVBAYTAlVTMQ4wDAYDVQQRDAU5ODEwNDEL -MAkGA1UECAwCV0ExEDAOBgNVBAcMB1NlYXR0bGUxEDAOBgNVBAkMB1N0ZSAzMDAx -FzAVBgNVBAkMDjcxIENvbHVtYmlhIFN0MSUwIwYDVQQKDBxUaGUgQml0Y29pbiBG -b3VuZGF0aW9uLCBJbmMuMSUwIwYDVQQDDBxUaGUgQml0Y29pbiBGb3VuZGF0aW9u -LCBJbmMuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw37Vrv9Gbku0 -+kuV0t89TuyxtAcmT7QE4GcwESKKjmkxfzD9a0qlhqk8GfQ+fw4DHNN+nLKNv7xB -bk6aS7J2v2DcXkOjrP99P9jqgTkp7MC04VtG3OqVRGB+gum0pptRovYZUQXIdkY7 -GJOok/NDagwKiiUe2V2meZ7UctsZNvYeilQdTgKIIhrMB9NowCOhT8ocVL4Ki55/ -l7hukJn3fueCM3fHTwY2/1gaGsOHoCkFRsD7vokjAVpiY+8rUgvHjb0gxgojiVGd -6a6/F5XJwKJacvUyN4Hfc2K5lRMQjTTmo4aWNWIa0iJ3TK9BHpdSLJBqerMPvmnM -kkapS+ZTNQIDAQABo4IBjTCCAYkwHwYDVR0jBBgwFoAUKZFg/4pN+uv5pmq4z/nm -S71JzhIwHQYDVR0OBBYEFONpQ+cV82URVe+V8G57377KxxexMA4GA1UdDwEB/wQE -AwIHgDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMBEGCWCGSAGG -+EIBAQQEAwIEEDBGBgNVHSAEPzA9MDsGDCsGAQQBsjEBAgEDAjArMCkGCCsGAQUF -BwIBFh1odHRwczovL3NlY3VyZS5jb21vZG8ubmV0L0NQUzBDBgNVHR8EPDA6MDig -NqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9SU0FDb2RlU2lnbmlu -Z0NBLmNybDB0BggrBgEFBQcBAQRoMGYwPgYIKwYBBQUHMAKGMmh0dHA6Ly9jcnQu -Y29tb2RvY2EuY29tL0NPTU9ET1JTQUNvZGVTaWduaW5nQ0EuY3J0MCQGCCsGAQUF -BzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB -AGnBSi9K/9rgTAyKFKrfGWSfNOwAghmsnsvpZSQ7QyoGWBFKSgCs/70kErl18oHA -g7Y8loQB1yukZmJaCa3OvGud7smn45TCh0TMf4EpP20Wxf4rMQTxwAatasHL3+vi -I+Nl5bsRZ09kWjvayqLII5upjS/yq0JfpmyGl5k2C/fIpztq0iOLvqWlXcL4+51r -cMUAfX6E6EaZQm//ikp+w2+7MEXTKguOuV3gwsrTy0DsvkZl4YDgx/FA4ImzXopv -d+3KJPLvO+OSBqUD3JPwXHnuJqGAbLBFyyCa/feGUjLlR8cxcNWLWdp4qxtoIUPG -3wTsC9YgrglS0F7FKMXlNRY= +MIIFcTCCBFmgAwIBAgIRALWcUnSOxv9FQW3xdaMDO6swDQYJKoZIhvcNAQELBQAw +fDELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMSQwIgYDVQQD +ExtTZWN0aWdvIFJTQSBDb2RlIFNpZ25pbmcgQ0EwHhcNMTkwMzI3MDAwMDAwWhcN +MjAwMzI2MjM1OTU5WjCBtDELMAkGA1UEBhMCQ0gxDTALBgNVBBEMBDgwMDUxCzAJ +BgNVBAgMAlpIMRAwDgYDVQQHDAdaw7xyaWNoMRcwFQYDVQQJDA5NYXR0ZW5nYXNz +ZSAyNzEuMCwGA1UECgwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjEuMCwGA1UEAwwlQml0Y29pbiBDb3JlIENvZGUgU2lnbmluZyBBc3NvY2lh +dGlvbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK04VDwiY1wxcW3E +WTTGmnbciCwETwC96DG4qcoH2PPNsVy3dfwGh0C02Qj2vL64IfwIGUFSgREvyjZk +CNhEuJO2e0nO0rKNNH5v/JO+P7/VYPZkF5a3uUz9ulmihULXioieHB/q0l6BmiJL ++cYaMVfidL9Y+IJwgiTqjnpRhv1Ik083SPsu6GcfQT9MJfY/+xse2EP0l4GfdFE6 +DRcWjiC8UHpfpGYcImzSFZZpbFbqoAyhueCl28QU4f8QAbS6BqNfaAK9MMACWDcK +eTz3C5JK6CiUxOnGIxilXhljuybFUjR4jGl5eTRpuPWk95NTTYS36q+bx/1nYelx +0n4nnDMCAwEAAaOCAbMwggGvMB8GA1UdIwQYMBaAFA7hOqhTOjHVir7Bu61nGgOF +rTQOMB0GA1UdDgQWBBRbN7ECrPCdVvh58enwy3Dix46h2jAOBgNVHQ8BAf8EBAMC +B4AwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDAzARBglghkgBhvhC +AQEEBAMCBBAwQAYDVR0gBDkwNzA1BgwrBgEEAbIxAQIBAwIwJTAjBggrBgEFBQcC +ARYXaHR0cHM6Ly9zZWN0aWdvLmNvbS9DUFMwQwYDVR0fBDwwOjA4oDagNIYyaHR0 +cDovL2NybC5zZWN0aWdvLmNvbS9TZWN0aWdvUlNBQ29kZVNpZ25pbmdDQS5jcmww +cwYIKwYBBQUHAQEEZzBlMD4GCCsGAQUFBzAChjJodHRwOi8vY3J0LnNlY3RpZ28u +Y29tL1NlY3RpZ29SU0FDb2RlU2lnbmluZ0NBLmNydDAjBggrBgEFBQcwAYYXaHR0 +cDovL29jc3Auc2VjdGlnby5jb20wKwYDVR0RBCQwIoEgam9uYXNAYml0Y29pbmNv +cmVjb2Rlc2lnbmluZy5vcmcwDQYJKoZIhvcNAQELBQADggEBAF/AIXcFBWCC2Red +SHN4Cvko5mdSkDNgzjVFc+OwAJ5RdOgbERde4PnHm3Qmrnx+uMetVnmrC8Fv1Iwb +kkR0bdbWBj6lF6zMsClIN6WJEfY+qfj1qi7wyucu+3OElYRC9bm5Lf0mEHQr8lJ1 +lGvAjPh+/hmxoVNbHFMZ1Ea+BrbjVwiSznt0gzdMh0CispBZKLWCIwRwi+hFjQrw +Z7RLH8HeCJ5Ojl/OTDQqh6AylQ7l9w9KHsUt4Jqy/AnCCyAj2/6xjdwnuo3tCZwb +g/9CydiAacD/83odphEeC2iBa+0wsj9bWmyYKY7S9n0u+wm3wBfZbSVMDDPk/la1 +3qCUDLk= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF4DCCA8igAwIBAgIQLnyHzA6TSlL+lP0ct800rzANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTMwNTA5 -MDAwMDAwWhcNMjgwNTA4MjM1OTU5WjB9MQswCQYDVQQGEwJHQjEbMBkGA1UECBMS -R3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFD -T01PRE8gQ0EgTGltaXRlZDEjMCEGA1UEAxMaQ09NT0RPIFJTQSBDb2RlIFNpZ25p -bmcgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCmmJBjd5E0f4rR -3elnMRHrzB79MR2zuWJXP5O8W+OfHiQyESdrvFGRp8+eniWzX4GoGA8dHiAwDvth -e4YJs+P9omidHCydv3Lj5HWg5TUjjsmK7hoMZMfYQqF7tVIDSzqwjiNLS2PgIpQ3 -e9V5kAoUGFEs5v7BEvAcP2FhCoyi3PbDMKrNKBh1SMF5WgjNu4xVjPfUdpA6M0ZQ -c5hc9IVKaw+A3V7Wvf2pL8Al9fl4141fEMJEVTyQPDFGy3CuB6kK46/BAW+QGiPi -XzjbxghdR7ODQfAuADcUuRKqeZJSzYcPe9hiKaR+ML0btYxytEjy4+gh+V5MYnmL -Agaff9ULAgMBAAGjggFRMIIBTTAfBgNVHSMEGDAWgBS7r34CPfqm8TyEjq3uOJjs -2TIy1DAdBgNVHQ4EFgQUKZFg/4pN+uv5pmq4z/nmS71JzhIwDgYDVR0PAQH/BAQD -AgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAwEwYDVR0lBAwwCgYIKwYBBQUHAwMwEQYD -VR0gBAowCDAGBgRVHSAAMEwGA1UdHwRFMEMwQaA/oD2GO2h0dHA6Ly9jcmwuY29t -b2RvY2EuY29tL0NPTU9ET1JTQUNlcnRpZmljYXRpb25BdXRob3JpdHkuY3JsMHEG -CCsGAQUFBwEBBGUwYzA7BggrBgEFBQcwAoYvaHR0cDovL2NydC5jb21vZG9jYS5j -b20vQ09NT0RPUlNBQWRkVHJ1c3RDQS5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9v -Y3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQwFAAOCAgEAAj8COcPu+Mo7id4M -bU2x8U6ST6/COCwEzMVjEasJY6+rotcCP8xvGcM91hoIlP8l2KmIpysQGuCbsQci -GlEcOtTh6Qm/5iR0rx57FjFuI+9UUS1SAuJ1CAVM8bdR4VEAxof2bO4QRHZXavHf -WGshqknUfDdOvf+2dVRAGDZXZxHNTwLk/vPa/HUX2+y392UJI0kfQ1eD6n4gd2HI -TfK7ZU2o94VFB696aSdlkClAi997OlE5jKgfcHmtbUIgos8MbAOMTM1zB5TnWo46 -BLqioXwfy2M6FafUFRunUkcyqfS/ZEfRqh9TTjIwc8Jvt3iCnVz/RrtrIh2IC/gb -qjSm/Iz13X9ljIwxVzHQNuxHoc/Li6jvHBhYxQZ3ykubUa9MCEp6j+KjUuKOjswm -5LLY5TjCqO3GgZw1a6lYYUoKl7RLQrZVnb6Z53BtWfhtKgx/GWBfDJqIbDCsUgmQ -Fhv/K53b0CDKieoofjKOGd97SDMe12X4rsn4gxSTdn1k0I7OvjV9/3IxTZ+evR5s -L6iPDAZQ+4wns3bJ9ObXwzTijIchhmH+v1V04SF3AwpobLvkyanmz1kl63zsRQ55 -ZmjoIs2475iFTZYRPAmK0H+8KCgT+2rKVI2SXM3CZZgGns5IW9S1N5NGQXwH3c/6 -Q++6Z2H/fUnguzB9XIDj5hY5S6c= +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 -MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT -EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR -Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR -6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X -pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC -9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV -/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf -Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z -+pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w -qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah -SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC -u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf -Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq -crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E -FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl -wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM -4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV -2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna -FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ -CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK -boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke -jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL -S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb -QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl -0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB -NVOFBkpdn627G190 +MIIF9TCCA92gAwIBAgIQHaJIMG+bJhjQguCWfTPTajANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgx +MTAyMDAwMDAwWhcNMzAxMjMxMjM1OTU5WjB8MQswCQYDVQQGEwJHQjEbMBkGA1UE +CBMSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRgwFgYDVQQK +Ew9TZWN0aWdvIExpbWl0ZWQxJDAiBgNVBAMTG1NlY3RpZ28gUlNBIENvZGUgU2ln +bmluZyBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIYijTKFehif +SfCWL2MIHi3cfJ8Uz+MmtiVmKUCGVEZ0MWLFEO2yhyemmcuVMMBW9aR1xqkOUGKl +UZEQauBLYq798PgYrKf/7i4zIPoMGYmobHutAMNhodxpZW0fbieW15dRhqb0J+V8 +aouVHltg1X7XFpKcAC9o95ftanK+ODtj3o+/bkxBXRIgCFnoOc2P0tbPBrRXBbZO +oT5Xax+YvMRi1hsLjcdmG0qfnYHEckC14l/vC0X/o84Xpi1VsLewvFRqnbyNVlPG +8Lp5UEks9wO5/i9lNfIi6iwHr0bZ+UYc3Ix8cSjz/qfGFN1VkW6KEQ3fBiSVfQ+n +oXw62oY1YdMCAwEAAaOCAWQwggFgMB8GA1UdIwQYMBaAFFN5v1qqK0rPVIDh2JvA +nfKyA2bLMB0GA1UdDgQWBBQO4TqoUzox1Yq+wbutZxoDha00DjAOBgNVHQ8BAf8E +BAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAUBggrBgEFBQcDAwYI +KwYBBQUHAwgwEQYDVR0gBAowCDAGBgRVHSAAMFAGA1UdHwRJMEcwRaBDoEGGP2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FDZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDB2BggrBgEFBQcBAQRqMGgwPwYIKwYBBQUHMAKGM2h0dHA6 +Ly9jcnQudXNlcnRydXN0LmNvbS9VU0VSVHJ1c3RSU0FBZGRUcnVzdENBLmNydDAl +BggrBgEFBQcwAYYZaHR0cDovL29jc3AudXNlcnRydXN0LmNvbTANBgkqhkiG9w0B +AQwFAAOCAgEATWNQ7Uc0SmGk295qKoyb8QAAHh1iezrXMsL2s+Bjs/thAIiaG20Q +BwRPvrjqiXgi6w9G7PNGXkBGiRL0C3danCpBOvzW9Ovn9xWVM8Ohgyi33i/klPeF +M4MtSkBIv5rCT0qxjyT0s4E307dksKYjalloUkJf/wTr4XRleQj1qZPea3FAmZa6 +ePG5yOLDCBaxq2NayBWAbXReSnV+pbjDbLXP30p5h1zHQE1jNfYw08+1Cg4LBH+g +S667o6XQhACTPlNdNKUANWlsvp8gJRANGftQkGG+OY96jk32nw4e/gdREmaDJhlI +lc5KycF/8zoFm/lv34h/wCOe0h5DekUxwZxNqfBZslkZ6GqNKQQCd3xLS81wvjqy +VVp4Pry7bwMQJXcVNIr5NsxDkuS6T/FikyglVyn7URnHoSVAaoRXxrKdsbwcCtp8 +Z359LukoTBh+xHsxQXGaSynsCz1XUNLK3f2eBVHlRHjdAd6xdZgNVCT98E7j4viD +vXK6yz067vBeF5Jobchh+abxKgoLpbn0nu6YMgWFnuv5gynTxix9vTp3Los3QqBq +gu07SqqUEKThDfgXxbZaeTMYkuO1dfih6Y4KJR7kHvGfWocj/5+kUZ77OYARzdu1 +xKeogG/lU9Tg46LC0lsa+jImLWpXcBw8pFguo/NbSwfcMlnzh6cabVg= -----END CERTIFICATE----- diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py deleted file mode 100644 index 66fdf7887f..0000000000 --- a/contrib/zmq/zmq_sub3.4.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -""" - ZMQ example using python3's asyncio - - Bitcoin should be started with the command line arguments: - bitcoind -testnet -daemon \ - -zmqpubrawtx=tcp://127.0.0.1:28332 \ - -zmqpubrawblock=tcp://127.0.0.1:28332 \ - -zmqpubhashtx=tcp://127.0.0.1:28332 \ - -zmqpubhashblock=tcp://127.0.0.1:28332 - - We use the asyncio library here. `self.handle()` installs itself as a - future at the end of the function. Since it never returns with the event - loop having an empty stack of futures, this creates an infinite loop. An - alternative is to wrap the contents of `handle` inside `while True`. - - The `@asyncio.coroutine` decorator and the `yield from` syntax found here - was introduced in python 3.4 and has been deprecated in favor of the `async` - and `await` keywords respectively. - - A blocking example using python 2.7 can be obtained from the git history: - https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py -""" - -import binascii -import asyncio -import zmq -import zmq.asyncio -import signal -import struct -import sys - -if (sys.version_info.major, sys.version_info.minor) < (3, 4): - print("This example only works with Python 3.4 and greater") - sys.exit(1) - -port = 28332 - -class ZMQHandler(): - def __init__(self): - self.loop = asyncio.get_event_loop() - self.zmqContext = zmq.asyncio.Context() - - self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0) - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashtx") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx") - self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % port) - - @asyncio.coroutine - def handle(self) : - msg = yield from self.zmqSubSocket.recv_multipart() - topic = msg[0] - body = msg[1] - sequence = "Unknown" - if len(msg[-1]) == 4: - msgSequence = struct.unpack('<I', msg[-1])[-1] - sequence = str(msgSequence) - if topic == b"hashblock": - print('- HASH BLOCK ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"hashtx": - print('- HASH TX ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"rawblock": - print('- RAW BLOCK HEADER ('+sequence+') -') - print(binascii.hexlify(body[:80])) - elif topic == b"rawtx": - print('- RAW TX ('+sequence+') -') - print(binascii.hexlify(body)) - # schedule ourselves to receive the next message - asyncio.ensure_future(self.handle()) - - def start(self): - self.loop.add_signal_handler(signal.SIGINT, self.stop) - self.loop.create_task(self.handle()) - self.loop.run_forever() - - def stop(self): - self.loop.stop() - self.zmqContext.destroy() - -daemon = ZMQHandler() -daemon.start() diff --git a/depends/README.md b/depends/README.md index 693bc36197..68a83a2aea 100644 --- a/depends/README.md +++ b/depends/README.md @@ -34,7 +34,7 @@ No other options are needed, the paths are automatically configured. #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools #### For Win32/Win64 cross compilation @@ -72,7 +72,7 @@ The following can be set when running make: make FOO=bar NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking - RAPIDCHECK: build rapidcheck (experimental) + RAPIDCHECK: build rapidcheck (experimental, requires cmake) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids diff --git a/depends/config.site.in b/depends/config.site.in index b7a5e795c8..52b9a7eca2 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -67,7 +67,7 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH +PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH if test -n "@AR@"; then AR=@AR@ diff --git a/depends/funcs.mk b/depends/funcs.mk index fbefc890ef..8f03c5f37a 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -76,8 +76,9 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands +# The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) -$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source) +$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= $(1)_build_cmds ?= $(1)_config_cmds ?= @@ -178,7 +179,7 @@ $($(1)_preprocessed): | $($(1)_extracted) $(AT)touch $$@ $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) $(AT)echo Configuring $(1)... - $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar xf $($(package)_cached); ) + $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) $(AT)mkdir -p $$(@D) $(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) $(AT)touch $$@ diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk index 5f247e9bf3..c3054cbd1a 100644 --- a/depends/packages/native_biplist.mk +++ b/depends/packages/native_biplist.mk @@ -3,13 +3,13 @@ $(package)_version=1.0.3 $(package)_download_path=https://bitbucket.org/wooster/biplist/downloads $(package)_file_name=biplist-$($(package)_version).tar.gz $(package)_sha256_hash=4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 44d238cc4c..ccd72a99bd 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -22,12 +22,12 @@ define $(package)_extract_cmds echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ - tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ + tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ rm -f toolchain/lib/libc++abi.so* && \ echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ chmod +x toolchain/bin/$(host)-dsymutil && \ - tar --strip-components=1 -xf $($(package)_source) + tar --no-same-owner --strip-components=1 -xf $($(package)_source) endef define $(package)_set_vars diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 116fa25d38..f99b689ecd 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -3,14 +3,14 @@ $(package)_version=1.1.2 $(package)_download_path=https://github.com/al45tair/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=3b3ecb7bf0a5157f5b6010bc3af7c141fb0ad3527084e63336220d22744bc20c -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages $(package)_dependencies=native_biplist define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 306c835656..e60b99dccc 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -3,13 +3,13 @@ $(package)_version=2.0.7 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=6f606d3b6bccd2112aeabf1a063f5b5ece87005a5d7e97c8faca23b916e88838 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_protobuf.mk b/depends/packages/native_protobuf.mk index ce50b366fa..1de8c37d36 100644 --- a/depends/packages/native_protobuf.mk +++ b/depends/packages/native_protobuf.mk @@ -5,7 +5,7 @@ $(package)_file_name=protobuf-$($(package)_version).tar.bz2 $(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910 define $(package)_set_vars -$(package)_config_opts=--disable-shared +$(package)_config_opts=--disable-shared --without-zlib endef define $(package)_config_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index deebf13e98..23cde9ee6d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -125,11 +125,11 @@ define $(package)_extract_cmds echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ - tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ mkdir qttranslations && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ mkdir qttools && \ - tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools + tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef define $(package)_preprocess_cmds @@ -138,7 +138,7 @@ define $(package)_preprocess_cmds sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ sed -i.old "s|X11/extensions/XIproto.h|X11/X.h|" qtbase/src/plugins/platforms/xcb/qxcbxsettings.cpp && \ - sed -i.old 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' qtbase/configure && \ + sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \ sed -i.old 's/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, 0)/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, kCGMouseButtonLeft)/' qtbase/src/plugins/platforms/cocoa/qcocoacursor.mm && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk index 19cf1cae2e..a35e091c80 100644 --- a/depends/packages/rapidcheck.mk +++ b/depends/packages/rapidcheck.mk @@ -1,18 +1,17 @@ package=rapidcheck -$(package)_version=10fc0cb -$(package)_download_path=https://github.com/MarcoFalke/rapidcheck/archive -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=9640926223c00af45bce4c7df8b756b5458a89b2ba74cfe3e404467f13ce26df +$(package)_version=3eb9b4ff69f4ff2d9932e8f852c2b2a61d7c20d3 +$(package)_download_path=https://github.com/emil-e/rapidcheck/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=5fbf82755c9a647127e62563be079448ff8b1db9ca80a52a673dd9a88fdb714b define $(package)_config_cmds - cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true . + cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_INSTALL_ALL_EXTRAS=ON endef define $(package)_build_cmds - $(MAKE) && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/include && \ - cp -a include/* $($(package)_staging_dir)$(host_prefix)/include/ && \ - cp -a extras/boost_test/include/rapidcheck/* $($(package)_staging_dir)$(host_prefix)/include/rapidcheck/ && \ - mkdir -p $($(package)_staging_dir)$(host_prefix)/lib && \ - cp -a librapidcheck.a $($(package)_staging_dir)$(host_prefix)/lib/ + $(MAKE) rapidcheck +endef + +define $(package)_stage_cmds + $(MAKE) rapidcheck install endef diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index 70f5dfc882..d22b6e8383 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,6 +1,6 @@ FreeBSD build guide ====================== -(updated for FreeBSD 11.1) +(updated for FreeBSD 12.0) This guide describes how to build bitcoind and command-line utilities on FreeBSD. @@ -10,55 +10,51 @@ This guide does not contain instructions for building the GUI. You will need the following dependencies, which can be installed as root via pkg: -``` +```shell pkg install autoconf automake boost-libs git gmake libevent libtool openssl pkgconf + +git clone https://github.com/bitcoin/bitcoin.git ``` In order to run the test suite (recommended), you will need to have Python 3 installed: -``` +```shell pkg install python3 ``` -For the wallet (optional): -``` -./contrib/install_db4.sh `pwd` -export BDB_PREFIX="$PWD/db4" -``` - See [dependencies.md](dependencies.md) for a complete overview. -Download the source code: -``` -git clone https://github.com/bitcoin/bitcoin +### Building BerkeleyDB + +BerkeleyDB is only necessary for the wallet functionality. To skip this, pass +`--disable-wallet` to `./configure` and skip to the next section. + +```shell +./contrib/install_db4.sh `pwd` +export BDB_PREFIX="$PWD/db4" ``` ## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error): -``` +With wallet: +```shell ./autogen.sh - -./configure # to build with wallet OR -./configure --disable-wallet # to build without wallet +./configure --with-gui=no \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` -followed by either: - -``` -gmake +Without wallet: +```shell +./autogen.sh +./configure --with-gui=no --disable-wallet ``` -to build without testing, or +followed by: +```shell +gmake # use -jX here for parallelism +gmake check # Run tests if Python 3 is available ``` -gmake check -``` - -to also run the test suite (recommended, if Python 3 is installed). - -*Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). -It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and -use the versioned gdb command (e.g. `gdb7111`). - diff --git a/doc/build-unix.md b/doc/build-unix.md index 522e3069ce..da65bc347a 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -78,19 +78,13 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in BerkeleyDB is required for the wallet. -**For Ubuntu only:** db4.8 packages are available [here](https://launchpad.net/~bitcoin/+archive/bitcoin). -You can add the repository and install using the following commands: - - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:bitcoin/bitcoin - sudo apt-get update - sudo apt-get install libdb4.8-dev libdb4.8++-dev - Ubuntu and Debian have their own libdb-dev and libdb++-dev packages, but these will install BerkeleyDB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. +Otherwise, you can build from self-compiled `depends` (see above). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) diff --git a/doc/build-windows.md b/doc/build-windows.md index 9641e0d3fd..036c585b44 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -71,6 +71,11 @@ If you want to build the windows installer with `make deploy` you need [NSIS](ht sudo apt install nsis +Acquire the source in the usual way: + + git clone https://github.com/bitcoin/bitcoin.git + cd bitcoin + ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: @@ -87,11 +92,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Once the source code is ready the build steps are below: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -115,11 +116,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Then build using: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends diff --git a/doc/dependencies.md b/doc/dependencies.md index 235eeb28b6..c445e2e23f 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -17,14 +17,30 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L65) | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L64) | +| librsvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L66) | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | -| Python (tests) | | [3.4](https://www.python.org/downloads) | | | | +| Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L86) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | [1.2.11](https://zlib.net/) | | | | No | + +Controlling dependencies +------------------------ +Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. + +#### Options passed to `./configure` +* MiniUPnPc is not needed with `--with-miniupnpc=no`. +* Berkeley DB is not needed with `--disable-wallet`. +* protobuf is not needed with `--disable-bip70`. +* Qt is not needed with `--without-gui`. +* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. +* ZeroMQ is needed only with the `--with-zmq` option. + +#### Other +* librsvg is only needed if you need to run `make deploy` on (cross-compliation to) macOS. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index f765346cd8..62c764bb31 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -144,7 +144,7 @@ For example, to describe a function use: */ bool function(int arg1, const char *arg2) ``` -A complete list of `@xxx` commands can be found at http://www.stack.nl/~dimitri/doxygen/manual/commands.html. +A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. @@ -185,7 +185,7 @@ Not OK (used plenty in the current source, but not picked up): // ``` -A full list of comment syntaxes picked up by Doxygen can be found at https://www.stack.nl/~dimitri/doxygen/manual/docblocks.html, +A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. @@ -952,8 +952,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: from there. - A RPC method must either be a wallet method or a non-wallet method. Do not - introduce new methods such as `signrawtransaction` that differ in behavior - based on presence of a wallet. + introduce new methods that differ in behavior based on presence of a wallet. - *Rationale*: as well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be diff --git a/doc/productivity.md b/doc/productivity.md index 862017290d..a93228ebdb 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -8,6 +8,7 @@ Table of Contents * [Cache compilations with `ccache`](#cache-compilations-with-ccache) * [Disable features with `./configure`](#disable-features-with-configure) * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) + * [Only build what you need](#only-build-what-you-need) * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Writing code](#writing-code) * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) @@ -32,6 +33,17 @@ Install `ccache` through your distribution's package manager, and run `./configu To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up. +To get the most out of ccache, put something like this in `~/.ccache/ccache.conf`: + +``` +max_size = 50.0G # or whatever cache size you prefer; default is 5G; 0 means unlimited +base_dir = /home/yourname # or wherever you keep your source files +``` + +Note: base_dir is required for ccache to share cached compiles of the same file across different repositories / paths; it will only do this for paths under base_dir. So this option is required for effective use of ccache with git worktrees (described below). + +You _must not_ set base_dir to "/", or anywhere that contains system headers (according to the ccache docs). + ### Disable features with `./configure` After running `./autogen.sh`, which generates the `./configure` file, use `./configure --help` to identify features that you can disable to save on compilation time. A few common flags: @@ -43,6 +55,8 @@ After running `./autogen.sh`, which generates the `./configure` file, use `./con --without-gui ``` +If you do need the wallet enabled, it is common for devs to add `--with-incompatible-bdb`. This uses your system bdb version for the wallet, so you don't have to find a copy of bdb 4.8. Wallets from such a build will be incompatible with any release binary (and vice versa), so use with caution on mainnet. + ### Make use of your threads with `make -j` If you have multiple threads on your machine, you can tell `make` to utilize all of them with: @@ -51,6 +65,20 @@ If you have multiple threads on your machine, you can tell `make` to utilize all make -j"$(($(nproc)+1))" ``` +### Only build what you need + +When rebuilding during development, note that running `make`, without giving a target, will do a lot of work you probably don't need. It will build the GUI (unless you've disabled it) and all the tests (which take much longer to build than the app does). + +Obviously, it is important to build and run the tests at appropriate times -- but when you just want a quick compile to check your work, consider picking one or a set of build targets relevant to what you're working on, e.g.: + +```sh +make src/bitcoind src/bitcoin-cli +make src/qt/bitcoin-qt +make -C src bitcoin_bench +``` + +(You can and should combine this with `-j`, as above, for a parallel build.) + ### Multiple working directories with `git worktrees` If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. @@ -76,7 +104,7 @@ Writing code ### Format C/C++/Protobuf diffs with `clang-format-diff.py` -See [contrib/devtools/README.md](contrib/devtools/README.md#clang-format-diff.py). +See [contrib/devtools/README.md](/contrib/devtools/README.md#clang-format-diff.py). ### Format Python diffs with `yapf-diff.py` @@ -136,7 +164,7 @@ This will add an `upstream-pull` remote to your git repository, which can be fet ### Diff the diffs with `git range-diff` -It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). `git range-diff` can help solve this problem by diffing the diffs. +It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). [git range-diff](https://git-scm.com/docs/git-range-diff) (Git >= 2.19) can help solve this problem by diffing the diffs. For example, to identify the differences between your previously reviewed diffs P1-5, and the new diffs P1-2,N3-4 as illustrated below: ``` @@ -152,7 +180,19 @@ You can do: git range-diff master previously-reviewed-head new-head ``` -Note that `git range-diff` also work for rebases. +Note that `git range-diff` also work for rebases: + +``` + P1--P2--P3--P4--P5 <-- previously-reviewed-head + / +...--m--m1--m2--m3 <-- master + \ + P1--P2--N3--N4 <-- new-head (with P3 modified, P4 & P5 squashed) + +PREV=P5 N=4 && git range-diff `git merge-base --all HEAD $PREV`...$PREV HEAD~$N...HEAD +``` + +Where `P5` is the commit you last reviewed and `4` is the number of commits in the new version. ----- diff --git a/doc/release-notes-14021.md b/doc/release-notes-14021.md deleted file mode 100644 index 4797a95bdb..0000000000 --- a/doc/release-notes-14021.md +++ /dev/null @@ -1,11 +0,0 @@ -Miscellaneous RPC Changes -------------------------- -- Descriptors with key origin information imported through `importmulti` will have their key origin information stored in the wallet for use with creating PSBTs. -- If `bip32derivs` of both `walletprocesspsbt` and `walletcreatefundedpsbt` is set to true but the key metadata for a public key has not been updated yet, then that key will have a derivation path as if it were just an independent key (i.e. no derivation path and its master fingerprint is itself) - -Miscellaneous Wallet changes ----------------------------- - -- The key metadata will need to be upgraded the first time that the HD seed is available. -For unencrypted wallets this will occur on wallet loading. -For encrypted wallets this will occur the first time the wallet is unlocked. diff --git a/doc/release-notes-14054.md b/doc/release-notes-14054.md new file mode 100644 index 0000000000..d8cad369c5 --- /dev/null +++ b/doc/release-notes-14054.md @@ -0,0 +1,7 @@ +P2P changes +----------- + +BIP 61 reject messages were deprecated in v0.18. They are now disabled by +default, but can be enabled by setting the `-enablebip61` command line option. +BIP 61 reject messages will be removed entirely in a future version of +Bitcoin Core. diff --git a/doc/release-notes-14481.md b/doc/release-notes-14481.md deleted file mode 100644 index ea8fc3c34e..0000000000 --- a/doc/release-notes-14481.md +++ /dev/null @@ -1,9 +0,0 @@ -Low-level RPC changes ----------------------- - -The `listunspent` RPC has been modified so that it also returns `witnessScript`, -the witness script in the case of a P2WSH or P2SH-P2WSH output. - -The `signrawtransactionwithkey` and `signrawtransactionwithwallet` RPCs have been -modified so that they also optionally accept a `witnessScript`, the witness script in the -case of a P2WSH or P2SH-P2WSH output. This is compatible with the change to `listunspent`. diff --git a/doc/release-notes-14491.md b/doc/release-notes-14491.md deleted file mode 100644 index 1cf36e85cf..0000000000 --- a/doc/release-notes-14491.md +++ /dev/null @@ -1,5 +0,0 @@ -Descriptor import support ---------------------- - -The `importmulti` RPC now supports importing of addresses from descriptors. A "desc" parameter can be provided instead of the "scriptPubKey" in a request, as well as an optional range for ranged descriptors to specify the start and end of the range to import. More information about -descriptors can be found [here](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). diff --git a/doc/release-notes-14667.md b/doc/release-notes-14667.md deleted file mode 100644 index 5cb1d0aee7..0000000000 --- a/doc/release-notes-14667.md +++ /dev/null @@ -1,4 +0,0 @@ -New RPC methods ------------- - -- `deriveaddresses` returns one or more addresses corresponding to an [output descriptor](/doc/descriptors.md). diff --git a/doc/release-notes-15226.md b/doc/release-notes-15226.md deleted file mode 100644 index 3be84db3e9..0000000000 --- a/doc/release-notes-15226.md +++ /dev/null @@ -1,8 +0,0 @@ -Miscellaneous RPC changes ------------- - -- The RPC `createwallet` now has an optional `blank` argument that can be used to create a blank wallet. -Blank wallets do not have any keys or HD seed. -They cannot be opened in software older than 0.18. -Once a blank wallet has a HD seed set (by using `sethdseed`) or private keys, scripts, addresses, and other watch only things have been imported, the wallet is no longer blank and can be opened in 0.17.x. -Encrypting a blank wallet will also set a HD seed for it. diff --git a/doc/release-notes-15393.md b/doc/release-notes-15393.md deleted file mode 100644 index f478dc798d..0000000000 --- a/doc/release-notes-15393.md +++ /dev/null @@ -1,4 +0,0 @@ -Dependencies ------------- - -- The minimum required version of QT has been increased from 5.2 to 5.5.1 (the [depends system](https://github.com/bitcoin/bitcoin/blob/master/depends/README.md) provides 5.9.7) diff --git a/doc/release-notes-15566.md b/doc/release-notes-15566.md new file mode 100644 index 0000000000..49964d7550 --- /dev/null +++ b/doc/release-notes-15566.md @@ -0,0 +1,3 @@ +Miscellaneous CLI Changes +------------------------- +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to `chain` and now returns the current network name as defined in BIP70 (main, test, regtest).
\ No newline at end of file diff --git a/doc/release-notes-15620.md b/doc/release-notes-15620.md new file mode 100644 index 0000000000..bf89a70a4e --- /dev/null +++ b/doc/release-notes-15620.md @@ -0,0 +1,13 @@ +Updated RPCs +------------ + +* The -maxtxfee setting no longer has any effect on non-wallet RPCs. + + The `sendrawtransaction` and `testmempoolaccept` RPC methods previously + accepted an `allowhighfees` parameter to fail the mempool acceptance in case + the transaction's fee would exceed the value of the command line argument + `-maxtxfee`. To uncouple the RPCs from the global option, they now have a + hardcoded default for the maximum transaction fee, that can be changed for + both RPCs on a per-call basis with the `maxfeerate` parameter. The + `allowhighfees` boolean option has been removed and replaced by the + `maxfeerate` numeric option. diff --git a/doc/release-notes-15637.md b/doc/release-notes-15637.md new file mode 100644 index 0000000000..048d5e7218 --- /dev/null +++ b/doc/release-notes-15637.md @@ -0,0 +1,3 @@ +RPC changes +----------- +In getmempoolancestors, getmempooldescendants, getmempoolentry and getrawmempool RPCs, to be consistent with the returned value and other RPCs such as getrawtransaction, vsize has been added and size is now deprecated. size will only be returned if bitcoind is started with `-deprecatedrpc=size`.
\ No newline at end of file diff --git a/doc/release-notes.md b/doc/release-notes.md index a6408cf1e6..a876b70ba7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -66,313 +66,27 @@ platform. Notable changes =============== -Mining ------- - -- Calls to `getblocktemplate` will fail if the segwit rule is not specified. - Calling `getblocktemplate` without segwit specified is almost certainly - a misconfiguration since doing so results in lower rewards for the miner. - Failed calls will produce an error message describing how to enable the - segwit rule. - -Configuration option changes ----------------------------- - -- A warning is printed if an unrecognized section name is used in the - configuration file. Recognized sections are `[test]`, `[main]`, and - `[regtest]`. - -- Four new options are available for configuring the maximum number of - messages that ZMQ will queue in memory (the "high water mark") before - dropping additional messages. The default value is 1,000, the same as - was used for previous releases. See the [ZMQ - documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/zmq.md#usage) - for details. - -- The `enablebip61` option (introduced in Bitcoin Core 0.17.0) is - used to toggle sending of BIP 61 reject messages. Reject messages have no use - case on the P2P network and are only logged for debugging by most network - nodes. The option will now by default be off for improved privacy and security - as well as reduced upload usage. The option can explicitly be turned on for - local-network debugging purposes. - -- The `rpcallowip` option can no longer be used to automatically listen - on all network interfaces. Instead, the `rpcbind` parameter must also - be used to specify the IP addresses to listen on. Listening for RPC - commands over a public network connection is insecure and should be - disabled, so a warning is now printed if a user selects such a - configuration. If you need to expose RPC in order to use a tool - like Docker, ensure you only bind RPC to your localhost, e.g. `docker - run [...] -p 127.0.0.1:8332:8332` (this is an extra `:8332` over the - normal Docker port specification). - -- The `rpcpassword` option now causes a startup error if the password - set in the configuration file contains a hash character (#), as it's - ambiguous whether the hash character is meant for the password or as a - comment. - -- The `whitelistforcerelay` option is used to relay transactions from - whitelisted peers even when not accepted to the mempool. This option now - defaults to being off, so that changes in policy and disconnect/ban behavior - will not cause a node that is whitelisting another to be dropped by peers. - Users can still explicitly enable this behavior with the command line option - (and may want to consider [contacting](https://bitcoincore.org/en/contact/) - the Bitcoin Core project to let us know about their - use-case, as this feature could be deprecated in the future). - -Documentation -------------- - -- A new short - [document](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) - about the JSON-RPC interface describes cases where the results of an - RPC might contain inconsistencies between data sourced from different - subsystems, such as wallet state and mempool state. A note is added - to the [REST interface documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) - indicating that the same rules apply. - -- Further information is added to the [JSON-RPC - documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) - about how to secure this interface. - -- A new [document](https://github.com/bitcoin/bitcoin/blob/master/doc/bitcoin-conf.md) - about the `bitcoin.conf` file describes how to use it to configure - Bitcoin Core. - -- A new document introduces Bitcoin Core's BIP174 - [Partially-Signed Bitcoin Transactions (PSBT)](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) - interface, which is used to allow multiple programs to collaboratively - work to create, sign, and broadcast new transactions. This is useful - for offline (cold storage) wallets, multisig wallets, coinjoin - implementations, and many other cases where two or more programs need - to interact to generate a complete transaction. - -- The [output script descriptor](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md) - documentation has been updated with information about new features in - this still-developing language for describing the output scripts that - a wallet or other program wants to receive notifications for, such as - which addresses it wants to know received payments. The language is - currently used in the `scantxoutset` RPC and is expected to be adapted - to other RPCs and to the underlying wallet structure. - -Build system changes --------------------- - -- A new `--disable-bip70` option may be passed to `./configure` to - prevent Bitcoin-Qt from being built with support for the BIP70 payment - protocol or from linking libssl. As the payment protocol has exposed - Bitcoin Core to libssl vulnerabilities in the past, builders who don't - need BIP70 support are encouraged to use this option to reduce their - exposure to future vulnerabilities. - -Deprecated or removed RPCs --------------------------- - -- The `signrawtransaction` RPC is removed after being deprecated and - hidden behind a special configuration option in version 0.17.0. - -- The 'account' API is removed after being deprecated in v0.17. The - 'label' API was introduced in v0.17 as a replacement for accounts. - See the [release notes from v0.17](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.17.0.md#label-and-account-apis-for-wallet) - for a full description of the changes from the 'account' API to the - 'label' API. - -- The `addwitnessaddress` RPC is removed after being deprecated in - version 0.13.0. - -- The wallet's `generate` RPC method is deprecated and will be fully - removed in a subsequent major version. This RPC is only used for - testing, but its implementation reached across multiple subsystems - (wallet and mining), so it is being deprecated to simplify the - wallet-node interface. Projects that are using `generate` for testing - purposes should transition to using the `generatetoaddress` RPC, which - does not require or use the wallet component. Calling - `generatetoaddress` with an address returned by the `getnewaddress` - RPC gives the same functionality as the old `generate` RPC. To - continue using `generate` in this version, restart bitcoind with the - `-deprecatedrpc=generate` configuration option. - -New RPCs --------- - -- The `getnodeaddresses` RPC returns peer addresses known to this - node. It may be used to find nodes to connect to without using a DNS - seeder. - -- The `listwalletdir` RPC returns a list of wallets in the wallet - directory (either the default wallet directory or the directory - configured by the `-walletdir` parameter). - -- The `getrpcinfo` returns runtime details of the RPC server. At the - moment, it returns an array of the currently active commands and how - long they've been running. - Updated RPCs ------------ -Note: some low-level RPC changes mainly useful for testing are described -in the Low-level Changes section below. - -- The `getpeerinfo` RPC now returns an additional `minfeefilter` field - set to the peer's BIP133 fee filter. You can use this to detect that - you have peers that are willing to accept transactions below the - default minimum relay fee. - -- The mempool RPCs, such as `getrawmempool` with `verbose=true`, now - return an additional "bip125-replaceable" value indicating whether the - transaction (or its unconfirmed ancestors) opts-in to asking nodes and - miners to replace it with a higher-feerate transaction spending any of - the same inputs. - -- The `settxfee` RPC previously silently ignored attempts to set the fee - below the allowed minimums. It now prints a warning. The special - value of "0" may still be used to request the minimum value. - -- The `getaddressinfo` RPC now provides an `ischange` field indicating - whether the wallet used the address in a change output. - -- The `importmulti` RPC has been updated to support P2WSH, P2WPKH, - P2SH-P2WPKH, and P2SH-P2WSH. Requests for P2WSH and P2SH-P2WSH accept - an additional `witnessscript` parameter. - -- The `importmulti` RPC now returns an additional `warnings` field for - each request with an array of strings explaining when fields are being - ignored or are inconsistent, if there are any. - -- The `getaddressinfo` RPC now returns an additional `solvable` boolean - field when Bitcoin Core knows enough about the address's scriptPubKey, - optional redeemScript, and optional witnessScript in order for the - wallet to be able to generate an unsigned input spending funds sent to - that address. - -- The `getaddressinfo`, `listunspent`, and `scantxoutset` RPCs now - return an additional `desc` field that contains an output descriptor - containing all key paths and signing information for the address - (except for the private key). The `desc` field is only returned for - `getaddressinfo` and `listunspent` when the address is solvable. - -- The `importprivkey` RPC will preserve previously-set labels for - addresses or public keys corresponding to the private key being - imported. For example, if you imported a watch-only address with the - label "cold wallet" in earlier releases of Bitcoin Core, subsequently - importing the private key would default to resetting the address's - label to the default empty-string label (""). In this release, the - previous label of "cold wallet" will be retained. If you optionally - specify any label besides the default when calling `importprivkey`, - the new label will be applied to the address. - -- See the [Mining](#mining) section for changes to `getblocktemplate`. - -- The `getmininginfo` RPC now omits `currentblockweight` and `currentblocktx` - when a block was never assembled via RPC on this node. - -- The `getrawtransaction` RPC & REST endpoints no longer check the - unspent UTXO set for a transaction. The remaining behaviors are as - follows: 1. If a blockhash is provided, check the corresponding block. - 2. If no blockhash is provided, check the mempool. 3. If no blockhash - is provided but txindex is enabled, also check txindex. - -- The `unloadwallet` RPC is now synchronous, meaning it will not return - until the wallet is fully unloaded. - -REST changes ------------- - -- A new `/rest/blockhashbyheight/` endpoint is added for fetching the - hash of the block in the current best blockchain based on its height - (how many blocks it is after the Genesis Block). - -Graphical User Interface (GUI) ------------------------------- - -- A new Window menu is added alongside the existing File, Settings, and - Help menus. Several items from the other menus that opened new - windows have been moved to this new Window menu. +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. -- In the Send tab, the checkbox for "pay only the required fee" - has been removed. Instead, the user can simply decrease the value in - the Custom Feerate field all the way down to the node's configured - minimum relay fee. +* The `sendmany` RPC had an argument `minconf` that was not well specified and + would lead to RPC errors even when the wallet's coin selection would succeed. + The `sendtoaddress` RPC never had this check, so to normalize the behavior, + `minconf` is now ignored in `sendmany`. If the coin selection does not + succeed due to missing coins, it will still throw an RPC error. Be reminded + that coin selection is influenced by the `-spendzeroconfchange`, + `-limitancestorcount`, `-limitdescendantcount` and `-walletrejectlongchains` + command line arguments. -- In the Overview tab, the watch-only balance will be the only - balance shown if the wallet was created using the `createwallet` RPC - and the `disable_private_keys` parameter was set to true. - -- The launch-on-startup option is no longer available on macOS if - compiled with macosx min version greater than 10.11 (use - CXXFLAGS="-mmacosx-version-min=10.11" - CFLAGS="-mmacosx-version-min=10.11" for setting the deployment - sdk version) - -Tools ----- - -- A new `bitcoin-wallet` tool is now distributed alongside Bitcoin - Core's other executables. Without needing to use any RPCs, this tool - can currently create a new wallet file or display some basic - information about an existing wallet, such as whether the wallet is - encrypted, whether it uses an HD seed, how many transactions it - contains, and how many address book entries it has. Low-level changes ================= -RPC ---- - -- The `submitblock` RPC previously returned the reason a rejected block - was invalid the first time it processed that block but returned a - generic "duplicate" rejection message on subsequent occasions it - processed the same block. It now always returns the fundamental - reason for rejecting an invalid block and only returns "duplicate" for - valid blocks it has already accepted. - -- A new `submitheader` RPC allows submitting block headers independently - from their block. This is likely only useful for testing. - -Configuration -------------- - -- The `-usehd` configuration option was removed in version 0.16. From - that version onwards, all new wallets created are hierarchical - deterministic wallets. This release makes specifying `-usehd` an - invalid configuration option. - -Network -------- - -- This release allows peers that your node automatically disconnected - for misbehavior (e.g. sending invalid data) to reconnect to your node - if you have unused incoming connection slots. If your slots fill up, - a misbehaving node will be disconnected to make room for nodes without - a history of problems (unless the misbehaving node helps your node in - some other way, such as by connecting to a part of the Internet from - which you don't have many other peers). Previously, Bitcoin Core - banned the IP addresses of misbehaving peers for a period of time - (default of 1 day); this was easily circumvented by attackers with - multiple IP addresses. If you manually ban a peer, such as by using - the `setban` RPC, all connections from that peer will still be - rejected. - -Security --------- - -- This release changes the Random Number Generator (RNG) used from - OpenSSL to Bitcoin Core's own implementation, although entropy - gathered by Bitcoin Core is fed out to OpenSSL and then read back in - when the program needs strong randomness. This moves Bitcoin Core a - little closer to no longer needing to depend on OpenSSL, a dependency - that has caused security issues in the past. - -Changes for particular platforms --------------------------------- - -- On macOS, Bitcoin Core now opts out of application CPU throttling - ("app nap") during initial blockchain download, when catching up from - over 100 blocks behind the current chain tip, or when reindexing chain - data. This helps prevent these operations from taking an excessively - long time because the operating system is attempting to conserve - power. +Example item +------------ Credits ======= @@ -380,4 +94,4 @@ Credits Thanks to everyone who directly contributed to this release: -As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-pr12255.md b/doc/release-notes/release-notes-pr12255.md deleted file mode 100644 index 5ac8b44283..0000000000 --- a/doc/release-notes/release-notes-pr12255.md +++ /dev/null @@ -1,17 +0,0 @@ -systemd init file -========= - -The systemd init file (`contrib/init/bitcoind.service`) has been changed to use -`/var/lib/bitcoind` as the data directory instead of `~bitcoin/.bitcoin`. This -change makes Bitcoin Core more consistent with other services, and makes the -systemd init config more consistent with existing Upstart and OpenRC configs. - -The configuration, PID, and data directories are now completely managed by -systemd, which will take care of their creation, permissions, etc. See -[`systemd.exec (5)`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=) -for more details. - -When using the provided init files under `contrib/init`, overriding the -`datadir` option in `/etc/bitcoin/bitcoin.conf` will have no effect. This is -because the command line arguments specified in the init files take precedence -over the options specified in `/etc/bitcoin/bitcoin.conf`. diff --git a/doc/release-process.md b/doc/release-process.md index d20a3dc6b3..eb1f5ad222 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -291,24 +291,48 @@ bitcoin.org (see below for bitcoin.org update instructions). - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well as update the OS download links. Ping @saivann/@harding (saivann/harding on Freenode) in case anything goes wrong -- Announce the release: - - - bitcoin-dev and bitcoin-core-dev mailing list - - - Bitcoin Core announcements list https://bitcoincore.org/en/list/announcements/join/ +- Update other repositories and websites for new version - bitcoincore.org blog post - bitcoincore.org RPC documentation update - - Update title of #bitcoin on Freenode IRC + - Update packaging repo - - Optionally twitter, reddit /r/Bitcoin, ... but this will usually sort out itself + - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) + + - Create a new branch for the major release "0.xx" (used to build the snap package) + + - Notify MarcoFalke so that he can start building the snap package + + - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging (Click "Import Now" to fetch the branch) + - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging/+ref/0.xx (Click "Create snap package") + - Name it "bitcoin-core-snap-0.xx" + - Leave owner and series as-is + - Select architectures that are compiled via gitian + - Leave "automatically build when branch changes" unticked + - Tick "automatically upload to store" + - Put "bitcoin-core" in the registered store package name field + - Tick the "edge" box + - Put "0.xx" in the track field + - Click "create snap package" + - Click "Request builds" for every new release on this branch (after updating the snapcraft.yml in the branch to reflect the latest gitian results) + - Promote release on https://snapcraft.io/bitcoin-core/releases if it passes sanity checks - - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) + - This repo - - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) + - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) - - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. + - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. + +- Announce the release: + + - bitcoin-dev and bitcoin-core-dev mailing list + + - Bitcoin Core announcements list https://bitcoincore.org/en/list/announcements/join/ + + - Update title of #bitcoin on Freenode IRC + + - Optionally twitter, reddit /r/Bitcoin, ... but this will usually sort out itself - Celebrate diff --git a/doc/translation_process.md b/doc/translation_process.md index 9692832842..b9a10b6527 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -8,7 +8,7 @@ Transifex is setup to monitor the GitHub repo for updates, and when code contain Multiple language support is critical in assisting Bitcoin’s global adoption, and growth. One of Bitcoin’s greatest strengths is cross-border money transfers, any help making that easier is greatly appreciated. -See the [Transifex Bitcoin project](https://www.transifex.com/projects/p/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. +See the [Transifex Bitcoin project](https://www.transifex.com/bitcoin/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. ### Writing code with translations We use automated scripts to help extract translations in both Qt, and non-Qt source files. It is rarely necessary to manually edit the files in `src/qt/locale/`. The translation source files must adhere to the following format: @@ -43,7 +43,7 @@ git commit ### Creating a Transifex account Visit the [Transifex Signup](https://www.transifex.com/signup/) page to create an account. Take note of your username and password, as they will be required to configure the command-line tool. -You can find the Bitcoin translation project at [https://www.transifex.com/projects/p/bitcoin/](https://www.transifex.com/projects/p/bitcoin/). +You can find the Bitcoin translation project at [https://www.transifex.com/bitcoin/bitcoin/](https://www.transifex.com/bitcoin/bitcoin/). ### Installing the Transifex client command-line tool The client is used to fetch updated translations. If you are having problems, or need more details, see [https://docs.transifex.com/client/installing-the-client](https://docs.transifex.com/client/installing-the-client) diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 6062d49c40..b5475dc1c6 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -4,6 +4,10 @@ # Network-related settings: +# Note that if you use testnet or regtest, particularly with the options +# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also +# want to read "[Sections]" further down. + # Run on the test network instead of the real bitcoin network. #testnet=0 @@ -53,6 +57,9 @@ # Listening mode, enabled by default except when 'connect' is being used #listen=1 +# Port on which to listen for connections (default: 8333, testnet: 18333, regtest: 18444) +#port= + # Maximum number of inbound+outbound connections. #maxconnections= @@ -115,6 +122,10 @@ # Wallet options +# Specify where to find wallet, lockfile and logs. If not present, those files will be +# created as new. +#wallet=</path/to/dir> + # Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). # This setting is over-ridden by the -paytxfee option. #txconfirmtarget=n @@ -142,3 +153,19 @@ # Minimize to the system tray #minimizetotray=1 + +# [Sections] +# Most options apply to mainnet, testnet and regtest. +# If you want to confine an option to just one network, you should add it in the +# relevant section below. +# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet +# only apply to mainnet unless they appear in the appropriate section below. + +# Options only for mainnet +[main] + +# Options only for testnet +[test] + +# Options only for regtest +[regtest] diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 046deed9ea..acf9e86667 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -5,7 +5,6 @@ SetCompressor /SOLID lzma # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" -!define VERSION @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@ !define COMPANY "@PACKAGE_NAME@ project" !define URL @PACKAGE_URL@ @@ -49,7 +48,7 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-${VERSION}-win@WINDOWS_BITS@-setup.exe +OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win@WINDOWS_BITS@-setup.exe !if "@WINDOWS_BITS@" == "64" InstallDir $PROGRAMFILES64\Bitcoin !else @@ -59,12 +58,12 @@ CRCCheck on XPStyle on BrandingText " " ShowInstDetails show -VIProductVersion ${VERSION}.@CLIENT_VERSION_BUILD@ +VIProductVersion @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@ VIAddVersionKey ProductName "@PACKAGE_NAME@" -VIAddVersionKey ProductVersion "${VERSION}" +VIAddVersionKey ProductVersion "@PACKAGE_VERSION@" VIAddVersionKey CompanyName "${COMPANY}" VIAddVersionKey CompanyWebsite "${URL}" -VIAddVersionKey FileVersion "${VERSION}" +VIAddVersionKey FileVersion "@PACKAGE_VERSION@" VIAddVersionKey FileDescription "" VIAddVersionKey LegalCopyright "" InstallDirRegKey HKCU "${REGKEY}" Path @@ -99,7 +98,7 @@ Section -post SEC0001 CreateShortcut "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" $INSTDIR\uninstall.exe !insertmacro MUI_STARTMENU_WRITE_END WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayName "$(^Name)" - WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "${VERSION}" + WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayVersion "@PACKAGE_VERSION@" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Publisher "${COMPANY}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" URLInfoAbout "${URL}" WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayIcon $INSTDIR\uninstall.exe diff --git a/src/Makefile.am b/src/Makefile.am index d491530ca1..0385c825ea 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -128,6 +128,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + flatfile.h \ fs.h \ httprpc.h \ httpserver.h \ @@ -153,6 +154,7 @@ BITCOIN_CORE_H = \ netaddress.h \ netbase.h \ netmessagemaker.h \ + node/coin.h \ node/transaction.h \ noui.h \ optional.h \ @@ -247,6 +249,7 @@ libbitcoin_server_a_SOURCES = \ chain.cpp \ checkpoints.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ @@ -260,6 +263,7 @@ libbitcoin_server_a_SOURCES = \ miner.cpp \ net.cpp \ net_processing.cpp \ + node/coin.cpp \ node/transaction.cpp \ noui.cpp \ outputtype.cpp \ @@ -342,6 +346,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha256.h \ crypto/hmac_sha512.cpp \ crypto/hmac_sha512.h \ + crypto/poly1305.h \ + crypto/poly1305.cpp \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ @@ -428,8 +434,8 @@ libbitcoin_common_a_SOURCES = \ netaddress.cpp \ netbase.cpp \ policy/feerate.cpp \ - psbt.cpp \ protocol.cpp \ + psbt.cpp \ scheduler.cpp \ script/descriptor.cpp \ script/ismine.cpp \ @@ -491,6 +497,8 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif +# Libraries below may be listed more than once to resolve circular dependencies (see +# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ @@ -566,12 +574,14 @@ bitcoin_wallet_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_ZMQ) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ - $(LIBSECP256K1) + $(LIBSECP256K1) \ + $(LIBUNIVALUE) -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # @@ -638,13 +648,13 @@ clean-local: check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) endif if ENABLE_BIP70 diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 5e787ca222..b84360e84b 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -26,10 +26,12 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/rpc_mempool.cpp \ bench/verify_script.cpp \ bench/base58.cpp \ bench/bech32.cpp \ bench/lockedpool.cpp \ + bench/poly1305.cpp \ bench/prevector.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) @@ -37,6 +39,7 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ @@ -47,7 +50,9 @@ bench_bench_bitcoin_LDADD = \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ - $(LIBUNIVALUE) + $(LIBUNIVALUE) \ + $(EVENT_PTHREADS_LIBS) \ + $(EVENT_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -57,7 +62,7 @@ if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 84bc326cfe..a6e0785616 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -51,14 +51,32 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ - test/test_bitcoin_main.cpp \ + test/main.cpp \ test/test_bitcoin.h \ test/test_bitcoin.cpp FUZZ_SUITE = \ + test/test_bitcoin.h \ + test/test_bitcoin.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h +FUZZ_SUITE_LD_COMMON = \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBUNIVALUE) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(BOOST_LIBS) \ + $(LIBMEMENV) \ + $(LIBSECP256K1) \ + $(EVENT_LIBS) \ + $(CRYPTO_LIBS) \ + $(EVENT_PTHREADS_LIBS) + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -83,6 +101,7 @@ BITCOIN_TESTS =\ test/cuckoocache_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ @@ -90,7 +109,7 @@ BITCOIN_TESTS =\ test/key_tests.cpp \ test/limitedmap_tests.cpp \ test/dbwrapper_tests.cpp \ - test/main_tests.cpp \ + test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ @@ -169,7 +188,7 @@ test_test_bitcoin_LDADD += $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDC test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ -test_test_bitcoin_LDADD += $(ZMQ_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ @@ -177,358 +196,127 @@ test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_block_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTRANSACTION_DESERIALIZE=1 test_fuzz_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_transaction_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_transaction_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocklocator_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocklocator_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockmerkleroot_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockmerkleroot_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_addrman_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_addrman_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_banentry_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_banentry_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_coins_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_coins_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_netaddr_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_netaddr_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_script_flags_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_script_flags_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_service_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_service_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_messageheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_messageheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_address_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_address_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_inv_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_inv_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_bloomfilter_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_bloomfilter_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_diskblockindex_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_diskblockindex_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txoutcompressor_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txoutcompressor_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactions_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactionsrequest_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) endif # ENABLE_FUZZ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrman.cpp b/src/addrman.cpp index 06c342ba73..8a5f78d1c5 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -239,7 +239,9 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime // Will moving this address into tried evict another entry? if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table, moving %s to m_tried_collisions=%d\n", addr.ToString(), m_tried_collisions.size()); + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { m_tried_collisions.insert(nId); } @@ -561,12 +563,19 @@ void CAddrMan::ResolveCollisions_() // Give address at least 60 seconds to successfully connect if (GetAdjustedTime() - info_old.nLastTry > 60) { - LogPrint(BCLog::ADDRMAN, "Swapping %s for %s in tried table\n", info_new.ToString(), info_old.ToString()); + LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); // Replaces an existing address already in the tried table with the new address Good_(info_new, false, GetAdjustedTime()); erase_collision = true; } + } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + // If the collision hasn't resolved in some reasonable amount of time, + // just evict the old entry -- we must not be able to + // connect to it for some reason. + LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; } } else { // Collision is not actually a collision anymore Good_(info_new, false, GetAdjustedTime()); diff --git a/src/addrman.h b/src/addrman.h index 003bd059f8..e54184ce35 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -166,6 +166,9 @@ public: //! the maximum number of tried addr collisions to store #define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 +//! the maximum time we'll spend trying to resolve a tried table collision, in seconds +static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes + /** * Stochastical (IP) address manager */ diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp new file mode 100644 index 0000000000..16342d0fbe --- /dev/null +++ b/src/bench/poly1305.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <iostream> + +#include <bench/bench.h> +#include <crypto/poly1305.h> + +/* Number of bytes to process per iteration */ +static constexpr uint64_t BUFFER_SIZE_TINY = 64; +static constexpr uint64_t BUFFER_SIZE_SMALL = 256; +static constexpr uint64_t BUFFER_SIZE_LARGE = 1024*1024; + +static void POLY1305(benchmark::State& state, size_t buffersize) +{ + std::vector<unsigned char> tag(POLY1305_TAGLEN, 0); + std::vector<unsigned char> key(POLY1305_KEYLEN, 0); + std::vector<unsigned char> in(buffersize, 0); + while (state.KeepRunning()) + poly1305_auth(tag.data(), in.data(), in.size(), key.data()); +} + +static void POLY1305_64BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_TINY); +} + +static void POLY1305_256BYTES(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_SMALL); +} + +static void POLY1305_1MB(benchmark::State& state) +{ + POLY1305(state, BUFFER_SIZE_LARGE); +} + +BENCHMARK(POLY1305_64BYTES, 500000); +BENCHMARK(POLY1305_256BYTES, 250000); +BENCHMARK(POLY1305_1MB, 340); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp new file mode 100644 index 0000000000..67d8a25564 --- /dev/null +++ b/src/bench/rpc_mempool.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <rpc/blockchain.h> +#include <txmempool.h> + +#include <univalue.h> + +#include <list> +#include <vector> + +static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp)); +} + +static void RpcMempool(benchmark::State& state) +{ + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + + for (int i = 0; i < 1000; ++i) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vin[0].scriptWitness.stack.push_back({1}); + tx.vout.resize(1); + tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx.vout[0].nValue = i; + const CTransactionRef tx_r{MakeTransactionRef(tx)}; + AddTx(tx_r, /* fee */ i, pool); + } + + while (state.KeepRunning()) { + (void)MempoolToJSON(pool, /*verbose*/ true); + } +} + +BENCHMARK(RpcMempool, 40); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index b0e1f67d93..1009a771f8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -253,16 +253,12 @@ public: } result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); - if (!batch[ID_WALLETINFO].isNull()) { - result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); - } result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); - result.pushKV("testnet", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"].get_str() == "test")); + result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO].isNull()) { result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index bcf24047ff..e15213c552 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -251,6 +251,8 @@ bool BlockFilter::BuildParams(GCSFilter::Params& params) const params.m_P = BASIC_FILTER_P; params.m_M = BASIC_FILTER_M; return true; + case BlockFilterType::INVALID: + return false; } return false; diff --git a/src/blockfilter.h b/src/blockfilter.h index 4d1f51dd60..e5e087ed5a 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -83,9 +83,10 @@ public: constexpr uint8_t BASIC_FILTER_P = 19; constexpr uint32_t BASIC_FILTER_M = 784931; -enum BlockFilterType : uint8_t +enum class BlockFilterType : uint8_t { BASIC = 0, + INVALID = 255, }; /** @@ -95,7 +96,7 @@ enum BlockFilterType : uint8_t class BlockFilter { private: - BlockFilterType m_filter_type; + BlockFilterType m_filter_type = BlockFilterType::INVALID; uint256 m_block_hash; GCSFilter m_filter; diff --git a/src/chain.h b/src/chain.h index 5a6f10b84f..2b6d2d082c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -8,6 +8,7 @@ #include <arith_uint256.h> #include <consensus/params.h> +#include <flatfile.h> #include <primitives/block.h> #include <tinyformat.h> #include <uint256.h> @@ -90,46 +91,6 @@ public: } }; -struct CDiskBlockPos -{ - int nFile; - unsigned int nPos; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } - - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return (a.nFile == b.nFile && a.nPos == b.nPos); - } - - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return !(a == b); - } - - void SetNull() { nFile = -1; nPos = 0; } - bool IsNull() const { return (nFile == -1); } - - std::string ToString() const - { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); - } - -}; - enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -266,8 +227,8 @@ public: nNonce = block.nNonce; } - CDiskBlockPos GetBlockPos() const { - CDiskBlockPos ret; + FlatFilePos GetBlockPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; ret.nPos = nDataPos; @@ -275,8 +236,8 @@ public: return ret; } - CDiskBlockPos GetUndoPos() const { - CDiskBlockPos ret; + FlatFilePos GetUndoPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; ret.nPos = nUndoPos; diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 820c9b93d9..6e7b4d3ded 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -17,6 +17,17 @@ # error "Bitcoin cannot be compiled without assertions." #endif +// Assumption: We assume a C++11 (ISO/IEC 14882:2011) compiler (minimum requirement). +// Example(s): We assume the presence of C++11 features everywhere :-) +// Note: MSVC does not report the expected __cplusplus value due to legacy +// reasons. +#if !defined(_MSC_VER) +// ISO Standard C++11 [cpp.predefined]p1: +// "The name __cplusplus is defined to the value 201103L when compiling a C++ +// translation unit." +static_assert(__cplusplus >= 201103L, "C++11 standard assumed"); +#endif + // Assumption: We assume the floating-point types to fulfill the requirements of // IEC 559 (IEEE 754) standard. // Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction @@ -40,8 +51,13 @@ static_assert(sizeof(double) == 8, "64-bit double assumed"); static_assert(sizeof(short) == 2, "16-bit short assumed"); static_assert(sizeof(int) == 4, "32-bit int assumed"); +// Assumption: We assume size_t to be 32-bit or 64-bit. +// Example(s): size_t assumed to be at least 32-bit in ecdsa_signature_parse_der_lax(...). +// size_t assumed to be 32-bit or 64-bit in MallocUsage(...). +static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, "size_t assumed to be 32-bit or 64-bit"); +static_assert(sizeof(size_t) == sizeof(void*), "Sizes of size_t and void* assumed to be equal"); + // Some important things we are NOT assuming (non-exhaustive list): -// * We are NOT assuming a specific value for sizeof(std::size_t). // * We are NOT assuming a specific value for std::endian::native. // * We are NOT assuming a specific value for std::locale("").name(). // * We are NOT assuming a specific value for std::numeric_limits<char>::is_signed. diff --git a/src/core_io.h b/src/core_io.h index ae377eb6e8..19fb7b29f6 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -16,7 +16,6 @@ class CBlockHeader; class CScript; class CTransaction; struct CMutableTransaction; -struct PartiallySignedTransaction; class uint256; class UniValue; @@ -37,11 +36,6 @@ bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); */ bool ParseHashStr(const std::string& strHex, uint256& result); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); - -//! Decode a base64ed PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); -//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); int ParseSighashString(const UniValue& sighash); // core_write.cpp diff --git a/src/core_read.cpp b/src/core_read.cpp index 536a7f4f17..a879a375ce 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -4,7 +4,6 @@ #include <core_io.h> -#include <psbt.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/script.h> @@ -177,33 +176,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) -{ - bool invalid; - std::string tx_data = DecodeBase64(base64_tx, &invalid); - if (invalid) { - error = "invalid base64"; - return false; - } - return DecodeRawPSBT(psbt, tx_data, error); -} - -bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) -{ - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); - try { - ss_data >> psbt; - if (!ss_data.empty()) { - error = "extra data after PSBT"; - return false; - } - } catch (const std::exception& e) { - error = e.what(); - return false; - } - return true; -} - bool ParseHashStr(const std::string& strHex, uint256& result) { if ((strHex.size() != 64) || !IsHex(strHex)) diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 919ea593b7..2dc2133434 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -12,36 +12,6 @@ extern "C" { #include <crypto/ctaes/ctaes.c> } -AES128Encrypt::AES128Encrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Encrypt::~AES128Encrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Encrypt::Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const -{ - AES128_encrypt(&ctx, 1, ciphertext, plaintext); -} - -AES128Decrypt::AES128Decrypt(const unsigned char key[16]) -{ - AES128_init(&ctx, key); -} - -AES128Decrypt::~AES128Decrypt() -{ - memset(&ctx, 0, sizeof(ctx)); -} - -void AES128Decrypt::Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const -{ - AES128_decrypt(&ctx, 1, plaintext, ciphertext); -} - AES256Encrypt::AES256Encrypt(const unsigned char key[32]) { AES256_init(&ctx, key); @@ -182,35 +152,3 @@ AES256CBCDecrypt::~AES256CBCDecrypt() { memset(iv, 0, sizeof(iv)); } - -AES128CBCEncrypt::AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : enc(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCEncrypt::~AES128CBCEncrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCEncrypt::Encrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCEncrypt(enc, iv, data, size, pad, out); -} - -AES128CBCDecrypt::AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) - : dec(key), pad(padIn) -{ - memcpy(iv, ivIn, AES_BLOCKSIZE); -} - -AES128CBCDecrypt::~AES128CBCDecrypt() -{ - memset(iv, 0, AES_BLOCKSIZE); -} - -int AES128CBCDecrypt::Decrypt(const unsigned char* data, int size, unsigned char* out) const -{ - return CBCDecrypt(dec, iv, data, size, pad, out); -} diff --git a/src/crypto/aes.h b/src/crypto/aes.h index fdad70c593..e06c8de272 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -12,33 +12,8 @@ extern "C" { } static const int AES_BLOCKSIZE = 16; -static const int AES128_KEYSIZE = 16; static const int AES256_KEYSIZE = 32; -/** An encryption class for AES-128. */ -class AES128Encrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Encrypt(const unsigned char key[16]); - ~AES128Encrypt(); - void Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const; -}; - -/** A decryption class for AES-128. */ -class AES128Decrypt -{ -private: - AES128_ctx ctx; - -public: - explicit AES128Decrypt(const unsigned char key[16]); - ~AES128Decrypt(); - void Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const; -}; - /** An encryption class for AES-256. */ class AES256Encrypt { @@ -89,30 +64,4 @@ private: unsigned char iv[AES_BLOCKSIZE]; }; -class AES128CBCEncrypt -{ -public: - AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCEncrypt(); - int Encrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Encrypt enc; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - -class AES128CBCDecrypt -{ -public: - AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); - ~AES128CBCDecrypt(); - int Decrypt(const unsigned char* data, int size, unsigned char* out) const; - -private: - const AES128Decrypt dec; - const bool pad; - unsigned char iv[AES_BLOCKSIZE]; -}; - #endif // BITCOIN_CRYPTO_AES_H diff --git a/src/crypto/poly1305.cpp b/src/crypto/poly1305.cpp new file mode 100644 index 0000000000..8a86c9601c --- /dev/null +++ b/src/crypto/poly1305.cpp @@ -0,0 +1,141 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation by Andrew Moon +// poly1305-donna-unrolled.c from https://github.com/floodyberry/poly1305-donna + +#include <crypto/common.h> +#include <crypto/poly1305.h> + +#include <string.h> + +#define mul32x32_64(a,b) ((uint64_t)(a) * (b)) + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, const unsigned char key[POLY1305_KEYLEN]) { + uint32_t t0,t1,t2,t3; + uint32_t h0,h1,h2,h3,h4; + uint32_t r0,r1,r2,r3,r4; + uint32_t s1,s2,s3,s4; + uint32_t b, nb; + size_t j; + uint64_t t[5]; + uint64_t f0,f1,f2,f3; + uint64_t g0,g1,g2,g3,g4; + uint64_t c; + unsigned char mp[16]; + + /* clamp key */ + t0 = ReadLE32(key+0); + t1 = ReadLE32(key+4); + t2 = ReadLE32(key+8); + t3 = ReadLE32(key+12); + + /* precompute multipliers */ + r0 = t0 & 0x3ffffff; t0 >>= 26; t0 |= t1 << 6; + r1 = t0 & 0x3ffff03; t1 >>= 20; t1 |= t2 << 12; + r2 = t1 & 0x3ffc0ff; t2 >>= 14; t2 |= t3 << 18; + r3 = t2 & 0x3f03fff; t3 >>= 8; + r4 = t3 & 0x00fffff; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + /* init state */ + h0 = 0; + h1 = 0; + h2 = 0; + h3 = 0; + h4 = 0; + + /* full blocks */ + if (inlen < 16) goto poly1305_donna_atmost15bytes; +poly1305_donna_16bytes: + m += 16; + inlen -= 16; + + t0 = ReadLE32(m-16); + t1 = ReadLE32(m-12); + t2 = ReadLE32(m-8); + t3 = ReadLE32(m-4); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8) | (1 << 24); + + +poly1305_donna_mul: + t[0] = mul32x32_64(h0,r0) + mul32x32_64(h1,s4) + mul32x32_64(h2,s3) + mul32x32_64(h3,s2) + mul32x32_64(h4,s1); + t[1] = mul32x32_64(h0,r1) + mul32x32_64(h1,r0) + mul32x32_64(h2,s4) + mul32x32_64(h3,s3) + mul32x32_64(h4,s2); + t[2] = mul32x32_64(h0,r2) + mul32x32_64(h1,r1) + mul32x32_64(h2,r0) + mul32x32_64(h3,s4) + mul32x32_64(h4,s3); + t[3] = mul32x32_64(h0,r3) + mul32x32_64(h1,r2) + mul32x32_64(h2,r1) + mul32x32_64(h3,r0) + mul32x32_64(h4,s4); + t[4] = mul32x32_64(h0,r4) + mul32x32_64(h1,r3) + mul32x32_64(h2,r2) + mul32x32_64(h3,r1) + mul32x32_64(h4,r0); + + h0 = (uint32_t)t[0] & 0x3ffffff; c = (t[0] >> 26); + t[1] += c; h1 = (uint32_t)t[1] & 0x3ffffff; b = (uint32_t)(t[1] >> 26); + t[2] += b; h2 = (uint32_t)t[2] & 0x3ffffff; b = (uint32_t)(t[2] >> 26); + t[3] += b; h3 = (uint32_t)t[3] & 0x3ffffff; b = (uint32_t)(t[3] >> 26); + t[4] += b; h4 = (uint32_t)t[4] & 0x3ffffff; b = (uint32_t)(t[4] >> 26); + h0 += b * 5; + + if (inlen >= 16) goto poly1305_donna_16bytes; + + /* final bytes */ +poly1305_donna_atmost15bytes: + if (!inlen) goto poly1305_donna_finish; + + for (j = 0; j < inlen; j++) mp[j] = m[j]; + mp[j++] = 1; + for (; j < 16; j++) mp[j] = 0; + inlen = 0; + + t0 = ReadLE32(mp+0); + t1 = ReadLE32(mp+4); + t2 = ReadLE32(mp+8); + t3 = ReadLE32(mp+12); + + h0 += t0 & 0x3ffffff; + h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; + h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; + h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; + h4 += (t3 >> 8); + + goto poly1305_donna_mul; + +poly1305_donna_finish: + b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; b = h1 >> 26; h1 = h1 & 0x3ffffff; + h2 += b; b = h2 >> 26; h2 = h2 & 0x3ffffff; + h3 += b; b = h3 >> 26; h3 = h3 & 0x3ffffff; + h4 += b; b = h4 >> 26; h4 = h4 & 0x3ffffff; + h0 += b * 5; b = h0 >> 26; h0 = h0 & 0x3ffffff; + h1 += b; + + g0 = h0 + 5; b = g0 >> 26; g0 &= 0x3ffffff; + g1 = h1 + b; b = g1 >> 26; g1 &= 0x3ffffff; + g2 = h2 + b; b = g2 >> 26; g2 &= 0x3ffffff; + g3 = h3 + b; b = g3 >> 26; g3 &= 0x3ffffff; + g4 = h4 + b - (1 << 26); + + b = (g4 >> 31) - 1; + nb = ~b; + h0 = (h0 & nb) | (g0 & b); + h1 = (h1 & nb) | (g1 & b); + h2 = (h2 & nb) | (g2 & b); + h3 = (h3 & nb) | (g3 & b); + h4 = (h4 & nb) | (g4 & b); + + f0 = ((h0 ) | (h1 << 26)) + (uint64_t)ReadLE32(&key[16]); + f1 = ((h1 >> 6) | (h2 << 20)) + (uint64_t)ReadLE32(&key[20]); + f2 = ((h2 >> 12) | (h3 << 14)) + (uint64_t)ReadLE32(&key[24]); + f3 = ((h3 >> 18) | (h4 << 8)) + (uint64_t)ReadLE32(&key[28]); + + WriteLE32(&out[ 0], f0); f1 += (f0 >> 32); + WriteLE32(&out[ 4], f1); f2 += (f1 >> 32); + WriteLE32(&out[ 8], f2); f3 += (f2 >> 32); + WriteLE32(&out[12], f3); +} diff --git a/src/crypto/poly1305.h b/src/crypto/poly1305.h new file mode 100644 index 0000000000..1598b013b9 --- /dev/null +++ b/src/crypto/poly1305.h @@ -0,0 +1,17 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_POLY1305_H +#define BITCOIN_CRYPTO_POLY1305_H + +#include <stdint.h> +#include <stdlib.h> + +#define POLY1305_KEYLEN 32 +#define POLY1305_TAGLEN 16 + +void poly1305_auth(unsigned char out[POLY1305_TAGLEN], const unsigned char *m, size_t inlen, + const unsigned char key[POLY1305_KEYLEN]); + +#endif // BITCOIN_CRYPTO_POLY1305_H diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 index 0000000000..8a8f7b681c --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdexcept> + +#include <flatfile.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : + m_dir(std::move(dir)), + m_prefix(prefix), + m_chunk_size(chunk_size) +{ + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +std::string FlatFilePos::ToString() const +{ + return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos); +} + +fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const +{ + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} + +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) +{ + if (pos.IsNull()) { + return nullptr; + } + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+"); + if (!file && !read_only) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + return file; +} + +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) +{ + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} + +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) +{ + FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 index 0000000000..374ceff411 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,96 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include <string> + +#include <fs.h> +#include <serialize.h> + +struct FlatFilePos +{ + int nFile; + unsigned int nPos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nPos)); + } + + FlatFilePos() : nFile(-1), nPos(0) {} + + FlatFilePos(int nFileIn, unsigned int nPosIn) : + nFile(nFileIn), + nPos(nPosIn) + {} + + friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { + return (a.nFile == b.nFile && a.nPos == b.nPos); + } + + friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { + return !(a == b); + } + + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } + + std::string ToString() const; +}; + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates + * access to and efficient management of these files. + */ +class FlatFileSeq +{ +private: + const fs::path m_dir; + const char* const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this amount. + */ + FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const FlatFilePos& pos) const; + + /** Open a handle to the file at the given position. */ + FILE* Open(const FlatFilePos& pos, bool read_only = false); + + /** + * Allocate additional space in a file after the given starting position. The amount allocated + * will be the minimum multiple of the sequence chunk size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const FlatFilePos& pos, bool finalize = false); +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/fs.cpp b/src/fs.cpp index 3c8f4c0247..f937f64e04 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -160,6 +160,7 @@ static std::string openmodeToStr(std::ios_base::openmode mode) void ifstream::open(const fs::path& p, std::ios_base::openmode mode) { close(); + mode |= std::ios_base::in; m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str()); if (m_file == nullptr) { return; @@ -183,6 +184,7 @@ void ifstream::close() void ofstream::open(const fs::path& p, std::ios_base::openmode mode) { close(); + mode |= std::ios_base::out; m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str()); if (m_file == nullptr) { return; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 10bc8419dd..7367ec7cb6 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public CDiskBlockPos +struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header @@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); + READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } CDiskTxPos() { @@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos } void SetNull() { - CDiskBlockPos::SetNull(); + FlatFilePos::SetNull(); nTxOffset = 0; } }; diff --git a/src/init.cpp b/src/init.cpp index 0013319ad5..6564ce5b9a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -53,6 +53,8 @@ #include <stdio.h> #ifndef WIN32 +#include <attributes.h> +#include <cerrno> #include <signal.h> #include <sys/stat.h> #endif @@ -92,6 +94,32 @@ std::unique_ptr<BanMan> g_banman; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +/** + * The PID file facilities. + */ +static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; + +static fs::path GetPidFile() +{ + return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); +} + +NODISCARD static bool CreatePidFile() +{ + FILE* file = fsbridge::fopen(GetPidFile(), "w"); + if (file) { +#ifdef WIN32 + fprintf(file, "%d\n", GetCurrentProcessId()); +#else + fprintf(file, "%d\n", getpid()); +#endif + fclose(file); + return true; + } else { + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); + } +} + ////////////////////////////////////////////////////////////////////////////// // // Shutdown @@ -260,13 +288,13 @@ void Shutdown(InitInterfaces& interfaces) } #endif -#ifndef WIN32 try { - fs::remove(GetPidFile()); + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } -#endif interfaces.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); @@ -352,7 +380,7 @@ void SetupServerArgs() gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), false, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS); gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS); - gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in MiB (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS); + gArgs.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS); gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS); @@ -364,11 +392,7 @@ void SetupServerArgs() gArgs.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)", -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), false, OptionsCategory::OPTIONS); gArgs.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), false, OptionsCategory::OPTIONS); -#ifndef WIN32 gArgs.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), false, OptionsCategory::OPTIONS); -#else - hidden_args.emplace_back("-pid"); -#endif gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), false, OptionsCategory::OPTIONS); @@ -476,7 +500,7 @@ void SetupServerArgs() gArgs.AddArg("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)", true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), true, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction or raw transaction; setting this too low may abort large transactions (default: %s)", + gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", // TODO move setting to wallet CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST); @@ -644,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (fReindex) { int nFile = 0; while (true) { - CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) + FlatFilePos pos(nFile, 0); + if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -815,7 +839,7 @@ void InitParameterInteraction() // Warn if unrecognized section name are present in the config file. for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf(_("Section [%s] is not recognized."), section)); + InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); } } @@ -1200,9 +1224,10 @@ bool AppInitMain(InitInterfaces& interfaces) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization -#ifndef WIN32 - CreatePidFile(GetPidFile(), getpid()); -#endif + if (!CreatePidFile()) { + // Detailed error printed inside CreatePidFile(). + return false; + } if (LogInstance().m_print_to_file) { if (gArgs.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { // Do this first since it both loads a bunch of debug.log into memory, @@ -1443,11 +1468,11 @@ bool AppInitMain(InitInterfaces& interfaces) uiInterface.InitMessage(_("Loading block index...")); - LOCK(cs_main); - do { const int64_t load_block_index_start_time = GetTimeMillis(); + bool is_coinsview_empty; try { + LOCK(cs_main); UnloadBlockIndex(); pcoinsTip.reset(); pcoinsdbview.reset(); @@ -1519,7 +1544,7 @@ bool AppInitMain(InitInterfaces& interfaces) // The on-disk coinsdb is now in a good state, create the cache pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); - bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); + is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); if (!is_coinsview_empty) { // LoadChainTip sets chainActive based on pcoinsTip's best block if (!LoadChainTip(chainparams)) { @@ -1528,18 +1553,25 @@ bool AppInitMain(InitInterfaces& interfaces) } assert(chainActive.Tip() != nullptr); } + } catch (const std::exception& e) { + LogPrintf("%s\n", e.what()); + strLoadError = _("Error opening block database"); + break; + } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on chainActive, and drops block data in - // mapBlockIndex based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...")); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); - break; - } + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on chainActive, and drops block data in + // mapBlockIndex based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...")); + if (!RewindBlockIndex(chainparams)) { + strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); + break; } + } + try { + LOCK(cs_main); if (!is_coinsview_empty) { uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { @@ -1644,11 +1676,11 @@ bool AppInitMain(InitInterfaces& interfaces) // ********************************************************* Step 11: import blocks - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { + if (!CheckDiskSpace(GetDataDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) { + if (!CheckDiskSpace(GetBlocksDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } diff --git a/src/interfaces/README.md b/src/interfaces/README.md index 57d41df746..f77d172153 100644 --- a/src/interfaces/README.md +++ b/src/interfaces/README.md @@ -2,9 +2,9 @@ The following interfaces are defined here: -* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437), [#14711](https://github.com/bitcoin/bitcoin/pull/14711), [#15288](https://github.com/bitcoin/bitcoin/pull/15288), and [#10973](https://github.com/bitcoin/bitcoin/pull/10973). -* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437). * [`Node`](node.h) — used by GUI to start & stop bitcoin node. Added in [#10244](https://github.com/bitcoin/bitcoin/pull/10244). diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index da810bc5e6..e409ced601 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -6,11 +6,29 @@ #include <chain.h> #include <chainparams.h> +#include <interfaces/handler.h> +#include <interfaces/wallet.h> +#include <net.h> +#include <node/coin.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> #include <primitives/block.h> +#include <primitives/transaction.h> +#include <protocol.h> +#include <rpc/protocol.h> +#include <rpc/server.h> +#include <shutdown.h> #include <sync.h> +#include <threadsafety.h> +#include <timedata.h> +#include <txmempool.h> +#include <ui_interface.h> #include <uint256.h> +#include <univalue.h> #include <util/system.h> #include <validation.h> +#include <validationinterface.h> #include <memory> #include <utility> @@ -132,6 +150,17 @@ class LockImpl : public Chain::Lock } return nullopt; } + bool checkFinalTx(const CTransaction& tx) override + { + LockAnnotation lock(::cs_main); + return CheckFinalTx(tx); + } + bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) override + { + LockAnnotation lock(::cs_main); + return AcceptToMemoryPool(::mempool, state, tx, nullptr /* missing inputs */, nullptr /* txn replaced */, + false /* bypass limits */, absurd_fee); + } }; class LockingStateImpl : public LockImpl, public UniqueLock<CCriticalSection> @@ -139,6 +168,94 @@ class LockingStateImpl : public LockImpl, public UniqueLock<CCriticalSection> using UniqueLock::UniqueLock; }; +class NotificationsHandlerImpl : public Handler, CValidationInterface +{ +public: + explicit NotificationsHandlerImpl(Chain& chain, Chain::Notifications& notifications) + : m_chain(chain), m_notifications(¬ifications) + { + RegisterValidationInterface(this); + } + ~NotificationsHandlerImpl() override { disconnect(); } + void disconnect() override + { + if (m_notifications) { + m_notifications = nullptr; + UnregisterValidationInterface(this); + } + } + void TransactionAddedToMempool(const CTransactionRef& tx) override + { + m_notifications->TransactionAddedToMempool(tx); + } + void TransactionRemovedFromMempool(const CTransactionRef& tx) override + { + m_notifications->TransactionRemovedFromMempool(tx); + } + void BlockConnected(const std::shared_ptr<const CBlock>& block, + const CBlockIndex* index, + const std::vector<CTransactionRef>& tx_conflicted) override + { + m_notifications->BlockConnected(*block, tx_conflicted); + } + void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + { + m_notifications->BlockDisconnected(*block); + } + void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->ChainStateFlushed(locator); } + void ResendWalletTransactions(int64_t best_block_time, CConnman*) override + { + // `cs_main` is always held when this method is called, so it is safe to + // call `assumeLocked`. This is awkward, and the `assumeLocked` method + // should be able to be removed entirely if `ResendWalletTransactions` + // is replaced by a wallet timer as suggested in + // https://github.com/bitcoin/bitcoin/issues/15619 + auto locked_chain = m_chain.assumeLocked(); + m_notifications->ResendWalletTransactions(*locked_chain, best_block_time); + } + Chain& m_chain; + Chain::Notifications* m_notifications; +}; + +class RpcHandlerImpl : public Handler +{ +public: + RpcHandlerImpl(const CRPCCommand& command) : m_command(command), m_wrapped_command(&command) + { + m_command.actor = [this](const JSONRPCRequest& request, UniValue& result, bool last_handler) { + if (!m_wrapped_command) return false; + try { + return m_wrapped_command->actor(request, result, last_handler); + } catch (const UniValue& e) { + // If this is not the last handler and a wallet not found + // exception was thrown, return false so the next handler can + // try to handle the request. Otherwise, reraise the exception. + if (!last_handler) { + const UniValue& code = e["code"]; + if (code.isNum() && code.get_int() == RPC_WALLET_NOT_FOUND) { + return false; + } + } + throw; + } + }; + ::tableRPC.appendCommand(m_command.name, &m_command); + } + + void disconnect() override final + { + if (m_wrapped_command) { + m_wrapped_command = nullptr; + ::tableRPC.removeCommand(m_command.name, &m_command); + } + } + + ~RpcHandlerImpl() override { disconnect(); } + + CRPCCommand m_command; + const CRPCCommand* m_wrapped_command; +}; + class ChainImpl : public Chain { public: @@ -172,13 +289,92 @@ public: } return true; } + void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(coins); } double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + RBFTransactionState isRBFOptIn(const CTransaction& tx) override + { + LOCK(::mempool.cs); + return IsRBFOptIn(tx, ::mempool); + } + bool hasDescendantsInMempool(const uint256& txid) override + { + LOCK(::mempool.cs); + auto it = ::mempool.GetIter(txid); + return it && (*it)->GetCountWithDescendants() > 1; + } + void relayTransaction(const uint256& txid) override + { + CInv inv(MSG_TX, txid); + g_connman->ForEachNode([&inv](CNode* node) { node->PushInventory(inv); }); + } + void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override + { + ::mempool.GetTransactionAncestry(txid, ancestors, descendants); + } + bool checkChainLimits(const CTransactionRef& tx) override + { + LockPoints lp; + CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); + CTxMemPool::setEntries ancestors; + auto limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + auto limit_ancestor_size = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + std::string unused_error_string; + LOCK(::mempool.cs); + return ::mempool.CalculateMemPoolAncestors(entry, ancestors, limit_ancestor_count, limit_ancestor_size, + limit_descendant_count, limit_descendant_size, unused_error_string); + } + CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override + { + return ::feeEstimator.estimateSmartFee(num_blocks, calc, conservative); + } + unsigned int estimateMaxBlocks() override + { + return ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + } + CFeeRate mempoolMinFee() override + { + return ::mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + } + CFeeRate relayMinFee() override { return ::minRelayTxFee; } + CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } + CFeeRate relayDustFee() override { return ::dustRelayFee; } + CAmount maxTxFee() override { return ::maxTxFee; } + bool getPruneMode() override { return ::fPruneMode; } + bool p2pEnabled() override { return g_connman != nullptr; } + bool isInitialBlockDownload() override { return IsInitialBlockDownload(); } + bool shutdownRequested() override { return ShutdownRequested(); } + int64_t getAdjustedTime() override { return GetAdjustedTime(); } + void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } + void initWarning(const std::string& message) override { InitWarning(message); } + void initError(const std::string& message) override { InitError(message); } + void loadWallet(std::unique_ptr<Wallet> wallet) override { ::uiInterface.LoadWallet(wallet); } + void showProgress(const std::string& title, int progress, bool resume_possible) override + { + ::uiInterface.ShowProgress(title, progress, resume_possible); + } + std::unique_ptr<Handler> handleNotifications(Notifications& notifications) override + { + return MakeUnique<NotificationsHandlerImpl>(*this, notifications); + } + void waitForNotifications() override { SyncWithValidationInterfaceQueue(); } + std::unique_ptr<Handler> handleRpc(const CRPCCommand& command) override + { + return MakeUnique<RpcHandlerImpl>(command); + } + void requestMempoolTransactions(Notifications& notifications) override + { + LOCK2(::cs_main, ::mempool.cs); + for (const CTxMemPoolEntry& entry : ::mempool.mapTx) { + notifications.TransactionAddedToMempool(entry.GetSharedTx()); + } + } }; - } // namespace std::unique_ptr<Chain> MakeChain() { return MakeUnique<ChainImpl>(); } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 3a54b9164e..d11a59241e 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -5,21 +5,58 @@ #ifndef BITCOIN_INTERFACES_CHAIN_H #define BITCOIN_INTERFACES_CHAIN_H -#include <optional.h> +#include <optional.h> // For Optional and nullopt +#include <primitives/transaction.h> // For CTransactionRef #include <memory> +#include <stddef.h> #include <stdint.h> #include <string> #include <vector> class CBlock; +class CFeeRate; +class CRPCCommand; class CScheduler; +class CValidationState; +class Coin; class uint256; +enum class RBFTransactionState; struct CBlockLocator; +struct FeeCalculation; namespace interfaces { -//! Interface for giving wallet processes access to blockchain state. +class Handler; +class Wallet; + +//! Interface giving clients (wallet processes, maybe other analysis tools in +//! the future) ability to access to the chain state, receive notifications, +//! estimate fees, and submit transactions. +//! +//! TODO: Current chain methods are too low level, exposing too much of the +//! internal workings of the bitcoin node, and not being very convenient to use. +//! Chain methods should be cleaned up and simplified over time. Examples: +//! +//! * The Chain::lock() method, which lets clients delay chain tip updates +//! should be removed when clients are able to respond to updates +//! asynchronously +//! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). +//! +//! * The isPotentialTip() and waitForNotifications() methods are too low-level +//! and should be replaced with a higher level +//! waitForNotificationsUpTo(block_hash) method that the wallet can call +//! instead +//! (https://github.com/bitcoin/bitcoin/pull/10973#discussion_r266995234). +//! +//! * The relayTransactions() and submitToMemoryPool() methods could be replaced +//! with a higher-level broadcastTransaction method +//! (https://github.com/bitcoin/bitcoin/pull/14978#issuecomment-459373984). +//! +//! * The initMessages() and loadWallet() methods which the wallet uses to send +//! notifications to the GUI should go away when GUI and wallet can directly +//! communicate with each other without going through the node +//! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). class Chain { public: @@ -102,6 +139,14 @@ public: //! is guaranteed to be an ancestor of the block used to create the //! locator. virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; + + //! Check if transaction will be final given chain height current time. + virtual bool checkFinalTx(const CTransaction& tx) = 0; + + //! Add transaction to memory pool if the transaction fee is below the + //! amount specified by absurd_fee. Returns false if the transaction + //! could not be added due to the fee or for another reason. + virtual bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) = 0; }; //! Return Lock interface. Chain is locked when this is called, and @@ -124,9 +169,116 @@ public: int64_t* time = nullptr, int64_t* max_time = nullptr) = 0; + //! Look up unspent output information. Returns coins in the mempool and in + //! the current chain UTXO set. Iterates through all the keys in the map and + //! populates the values. + virtual void findCoins(std::map<COutPoint, Coin>& coins) = 0; + //! Estimate fraction of total transactions verified if blocks up to //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + + //! Check if transaction is RBF opt in. + virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; + + //! Check if transaction has descendants in mempool. + virtual bool hasDescendantsInMempool(const uint256& txid) = 0; + + //! Relay transaction. + virtual void relayTransaction(const uint256& txid) = 0; + + //! Calculate mempool ancestor and descendant counts for the given transaction. + virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) = 0; + + //! Check if transaction will pass the mempool's chain limits. + virtual bool checkChainLimits(const CTransactionRef& tx) = 0; + + //! Estimate smart fee. + virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc = nullptr) = 0; + + //! Fee estimator max target. + virtual unsigned int estimateMaxBlocks() = 0; + + //! Mempool minimum fee. + virtual CFeeRate mempoolMinFee() = 0; + + //! Relay current minimum fee (from -minrelaytxfee and -incrementalrelayfee settings). + virtual CFeeRate relayMinFee() = 0; + + //! Relay incremental fee setting (-incrementalrelayfee), reflecting cost of relay. + virtual CFeeRate relayIncrementalFee() = 0; + + //! Relay dust fee setting (-dustrelayfee), reflecting lowest rate it's economical to spend. + virtual CFeeRate relayDustFee() = 0; + + //! Node max tx fee setting (-maxtxfee). + //! This could be replaced by a per-wallet max fee, as proposed at + //! https://github.com/bitcoin/bitcoin/issues/15355 + //! But for the time being, wallets call this to access the node setting. + virtual CAmount maxTxFee() = 0; + + //! Check if pruning is enabled. + virtual bool getPruneMode() = 0; + + //! Check if p2p enabled. + virtual bool p2pEnabled() = 0; + + //! Check if in IBD. + virtual bool isInitialBlockDownload() = 0; + + //! Check if shutdown requested. + virtual bool shutdownRequested() = 0; + + //! Get adjusted time. + virtual int64_t getAdjustedTime() = 0; + + //! Send init message. + virtual void initMessage(const std::string& message) = 0; + + //! Send init warning. + virtual void initWarning(const std::string& message) = 0; + + //! Send init error. + virtual void initError(const std::string& message) = 0; + + //! Send wallet load notification to the GUI. + virtual void loadWallet(std::unique_ptr<Wallet> wallet) = 0; + + //! Send progress indicator. + virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; + + //! Chain notifications. + class Notifications + { + public: + virtual ~Notifications() {} + virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} + virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} + virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} + virtual void BlockDisconnected(const CBlock& block) {} + virtual void ChainStateFlushed(const CBlockLocator& locator) {} + virtual void ResendWalletTransactions(Lock& locked_chain, int64_t best_block_time) {} + }; + + //! Register handler for notifications. + virtual std::unique_ptr<Handler> handleNotifications(Notifications& notifications) = 0; + + //! Wait for pending notifications to be handled. + virtual void waitForNotifications() = 0; + + //! Register handler for RPC. Command is not copied, so reference + //! needs to remain valid until Handler is disconnected. + virtual std::unique_ptr<Handler> handleRpc(const CRPCCommand& command) = 0; + + //! Synchronously send TransactionAddedToMempool notifications about all + //! current mempool transactions to the specified handler and return after + //! the last one is sent. These notifications aren't coordinated with async + //! notifications sent by handleNotifications, so out of date async + //! notifications from handleNotifications can arrive during and after + //! synchronous notifications from requestMempoolTransactions. Clients need + //! to be prepared to handle this by ignoring notifications about unknown + //! removed transactions and already added new transactions. + virtual void requestMempoolTransactions(Notifications& notifications) = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 96bde7e9f2..6f7dce0c24 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -275,7 +275,7 @@ public: } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::shared_ptr<CWallet> wallet) { fn(MakeWallet(wallet)); })); + return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::unique_ptr<Wallet>& wallet) { fn(std::move(wallet)); })); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 5b5430037c..60173b29ac 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -47,8 +47,6 @@ public: const CTransaction& get() override { return *m_tx; } - int64_t getVirtualSize() override { return GetVirtualTransactionSize(*m_tx); } - bool commit(WalletValueMap value_map, WalletOrderForm order_form, std::string& reject_reason) override @@ -56,7 +54,7 @@ public: auto locked_chain = m_wallet.chain().lock(); LOCK(m_wallet.cs_wallet); CValidationState state; - if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, g_connman.get(), state)) { + if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, state)) { reject_reason = state.GetRejectReason(); return false; } @@ -99,17 +97,13 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co //! Construct wallet tx status struct. WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - WalletTxStatus result; - auto mi = ::mapBlockIndex.find(wtx.hashBlock); - CBlockIndex* block = mi != ::mapBlockIndex.end() ? mi->second : nullptr; - result.block_height = (block ? block->nHeight : std::numeric_limits<int>::max()); + result.block_height = locked_chain.getBlockHeight(wtx.hashBlock).get_value_or(std::numeric_limits<int>::max()); result.blocks_to_maturity = wtx.GetBlocksToMaturity(locked_chain); result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; - result.is_final = CheckFinalTx(*wtx.tx); + result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); @@ -135,55 +129,55 @@ WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, class WalletImpl : public Wallet { public: - explicit WalletImpl(const std::shared_ptr<CWallet>& wallet) : m_shared_wallet(wallet), m_wallet(*wallet.get()) {} + explicit WalletImpl(const std::shared_ptr<CWallet>& wallet) : m_wallet(wallet) {} bool encryptWallet(const SecureString& wallet_passphrase) override { - return m_wallet.EncryptWallet(wallet_passphrase); + return m_wallet->EncryptWallet(wallet_passphrase); } - bool isCrypted() override { return m_wallet.IsCrypted(); } - bool lock() override { return m_wallet.Lock(); } - bool unlock(const SecureString& wallet_passphrase) override { return m_wallet.Unlock(wallet_passphrase); } - bool isLocked() override { return m_wallet.IsLocked(); } + bool isCrypted() override { return m_wallet->IsCrypted(); } + bool lock() override { return m_wallet->Lock(); } + bool unlock(const SecureString& wallet_passphrase) override { return m_wallet->Unlock(wallet_passphrase); } + bool isLocked() override { return m_wallet->IsLocked(); } bool changeWalletPassphrase(const SecureString& old_wallet_passphrase, const SecureString& new_wallet_passphrase) override { - return m_wallet.ChangeWalletPassphrase(old_wallet_passphrase, new_wallet_passphrase); + return m_wallet->ChangeWalletPassphrase(old_wallet_passphrase, new_wallet_passphrase); } - void abortRescan() override { m_wallet.AbortRescan(); } - bool backupWallet(const std::string& filename) override { return m_wallet.BackupWallet(filename); } - std::string getWalletName() override { return m_wallet.GetName(); } + void abortRescan() override { m_wallet->AbortRescan(); } + bool backupWallet(const std::string& filename) override { return m_wallet->BackupWallet(filename); } + std::string getWalletName() override { return m_wallet->GetName(); } bool getKeyFromPool(bool internal, CPubKey& pub_key) override { - return m_wallet.GetKeyFromPool(pub_key, internal); + return m_wallet->GetKeyFromPool(pub_key, internal); } - bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet.GetPubKey(address, pub_key); } - bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet.GetKey(address, key); } - bool isSpendable(const CTxDestination& dest) override { return IsMine(m_wallet, dest) & ISMINE_SPENDABLE; } - bool haveWatchOnly() override { return m_wallet.HaveWatchOnly(); }; + bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet->GetPubKey(address, pub_key); } + bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet->GetKey(address, key); } + bool isSpendable(const CTxDestination& dest) override { return IsMine(*m_wallet, dest) & ISMINE_SPENDABLE; } + bool haveWatchOnly() override { return m_wallet->HaveWatchOnly(); }; bool setAddressBook(const CTxDestination& dest, const std::string& name, const std::string& purpose) override { - return m_wallet.SetAddressBook(dest, name, purpose); + return m_wallet->SetAddressBook(dest, name, purpose); } bool delAddressBook(const CTxDestination& dest) override { - return m_wallet.DelAddressBook(dest); + return m_wallet->DelAddressBook(dest); } bool getAddress(const CTxDestination& dest, std::string* name, isminetype* is_mine, std::string* purpose) override { - LOCK(m_wallet.cs_wallet); - auto it = m_wallet.mapAddressBook.find(dest); - if (it == m_wallet.mapAddressBook.end()) { + LOCK(m_wallet->cs_wallet); + auto it = m_wallet->mapAddressBook.find(dest); + if (it == m_wallet->mapAddressBook.end()) { return false; } if (name) { *name = it->second.name; } if (is_mine) { - *is_mine = IsMine(m_wallet, dest); + *is_mine = IsMine(*m_wallet, dest); } if (purpose) { *purpose = it->second.purpose; @@ -192,52 +186,52 @@ public: } std::vector<WalletAddress> getAddresses() override { - LOCK(m_wallet.cs_wallet); + LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; - for (const auto& item : m_wallet.mapAddressBook) { - result.emplace_back(item.first, IsMine(m_wallet, item.first), item.second.name, item.second.purpose); + for (const auto& item : m_wallet->mapAddressBook) { + result.emplace_back(item.first, IsMine(*m_wallet, item.first), item.second.name, item.second.purpose); } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet.LearnRelatedScripts(key, type); } + void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { - LOCK(m_wallet.cs_wallet); - return m_wallet.AddDestData(dest, key, value); + LOCK(m_wallet->cs_wallet); + return m_wallet->AddDestData(dest, key, value); } bool eraseDestData(const CTxDestination& dest, const std::string& key) override { - LOCK(m_wallet.cs_wallet); - return m_wallet.EraseDestData(dest, key); + LOCK(m_wallet->cs_wallet); + return m_wallet->EraseDestData(dest, key); } std::vector<std::string> getDestValues(const std::string& prefix) override { - LOCK(m_wallet.cs_wallet); - return m_wallet.GetDestValues(prefix); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetDestValues(prefix); } void lockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.LockCoin(output); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->LockCoin(output); } void unlockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.UnlockCoin(output); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->UnlockCoin(output); } bool isLockedCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsLockedCoin(output.hash, output.n); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsLockedCoin(output.hash, output.n); } void listLockedCoins(std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.ListLockedCoins(outputs); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->ListLockedCoins(outputs); } std::unique_ptr<PendingWalletTx> createTransaction(const std::vector<CRecipient>& recipients, const CCoinControl& coin_control, @@ -246,25 +240,25 @@ public: CAmount& fee, std::string& fail_reason) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto pending = MakeUnique<PendingWalletTxImpl>(m_wallet); - if (!m_wallet.CreateTransaction(*locked_chain, recipients, pending->m_tx, pending->m_key, fee, change_pos, + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto pending = MakeUnique<PendingWalletTxImpl>(*m_wallet); + if (!m_wallet->CreateTransaction(*locked_chain, recipients, pending->m_tx, pending->m_key, fee, change_pos, fail_reason, coin_control, sign)) { return {}; } return std::move(pending); } - bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet.TransactionCanBeAbandoned(txid); } + bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet->TransactionCanBeAbandoned(txid); } bool abandonTransaction(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.AbandonTransaction(*locked_chain, txid); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->AbandonTransaction(*locked_chain, txid); } bool transactionCanBeBumped(const uint256& txid) override { - return feebumper::TransactionCanBeBumped(&m_wallet, txid); + return feebumper::TransactionCanBeBumped(m_wallet.get(), txid); } bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, @@ -274,46 +268,46 @@ public: CAmount& new_fee, CMutableTransaction& mtx) override { - return feebumper::CreateTransaction(&m_wallet, txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == + return feebumper::CreateTransaction(m_wallet.get(), txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == feebumper::Result::OK; } - bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(&m_wallet, mtx); } + bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(m_wallet.get(), mtx); } bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) override { - return feebumper::CommitTransaction(&m_wallet, txid, std::move(mtx), errors, bumped_txid) == + return feebumper::CommitTransaction(m_wallet.get(), txid, std::move(mtx), errors, bumped_txid) == feebumper::Result::OK; } CTransactionRef getTx(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { return mi->second.tx; } return {}; } WalletTx getWalletTx(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { - return MakeWalletTx(*locked_chain, m_wallet, mi->second); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { + return MakeWalletTx(*locked_chain, *m_wallet, mi->second); } return {}; } std::vector<WalletTx> getWalletTxs() override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); std::vector<WalletTx> result; - result.reserve(m_wallet.mapWallet.size()); - for (const auto& entry : m_wallet.mapWallet) { - result.emplace_back(MakeWalletTx(*locked_chain, m_wallet, entry.second)); + result.reserve(m_wallet->mapWallet.size()); + for (const auto& entry : m_wallet->mapWallet) { + result.emplace_back(MakeWalletTx(*locked_chain, *m_wallet, entry.second)); } return result; } @@ -322,16 +316,16 @@ public: int& num_blocks, int64_t& block_time) override { - auto locked_chain = m_wallet.chain().lock(true /* try_lock */); + auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) { return false; } - TRY_LOCK(m_wallet.cs_wallet, locked_wallet); + TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } - auto mi = m_wallet.mapWallet.find(txid); - if (mi == m_wallet.mapWallet.end()) { + auto mi = m_wallet->mapWallet.find(txid); + if (mi == m_wallet->mapWallet.end()) { return false; } if (Optional<int> height = locked_chain->getHeight()) { @@ -350,37 +344,37 @@ public: bool& in_mempool, int& num_blocks) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { num_blocks = locked_chain->getHeight().get_value_or(-1); in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; tx_status = MakeWalletTxStatus(*locked_chain, mi->second); - return MakeWalletTx(*locked_chain, m_wallet, mi->second); + return MakeWalletTx(*locked_chain, *m_wallet, mi->second); } return {}; } WalletBalances getBalances() override { WalletBalances result; - result.balance = m_wallet.GetBalance(); - result.unconfirmed_balance = m_wallet.GetUnconfirmedBalance(); - result.immature_balance = m_wallet.GetImmatureBalance(); - result.have_watch_only = m_wallet.HaveWatchOnly(); + result.balance = m_wallet->GetBalance(); + result.unconfirmed_balance = m_wallet->GetUnconfirmedBalance(); + result.immature_balance = m_wallet->GetImmatureBalance(); + result.have_watch_only = m_wallet->HaveWatchOnly(); if (result.have_watch_only) { - result.watch_only_balance = m_wallet.GetBalance(ISMINE_WATCH_ONLY); - result.unconfirmed_watch_only_balance = m_wallet.GetUnconfirmedWatchOnlyBalance(); - result.immature_watch_only_balance = m_wallet.GetImmatureWatchOnlyBalance(); + result.watch_only_balance = m_wallet->GetBalance(ISMINE_WATCH_ONLY); + result.unconfirmed_watch_only_balance = m_wallet->GetUnconfirmedWatchOnlyBalance(); + result.immature_watch_only_balance = m_wallet->GetImmatureWatchOnlyBalance(); } return result; } bool tryGetBalances(WalletBalances& balances, int& num_blocks) override { - auto locked_chain = m_wallet.chain().lock(true /* try_lock */); + auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) return false; - TRY_LOCK(m_wallet.cs_wallet, locked_wallet); + TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } @@ -388,68 +382,68 @@ public: num_blocks = locked_chain->getHeight().get_value_or(-1); return true; } - CAmount getBalance() override { return m_wallet.GetBalance(); } + CAmount getBalance() override { return m_wallet->GetBalance(); } CAmount getAvailableBalance(const CCoinControl& coin_control) override { - return m_wallet.GetAvailableBalance(&coin_control); + return m_wallet->GetAvailableBalance(&coin_control); } isminetype txinIsMine(const CTxIn& txin) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsMine(txin); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(txin); } isminetype txoutIsMine(const CTxOut& txout) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsMine(txout); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(txout); } CAmount getDebit(const CTxIn& txin, isminefilter filter) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.GetDebit(txin, filter); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetDebit(txin, filter); } CAmount getCredit(const CTxOut& txout, isminefilter filter) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.GetCredit(txout, filter); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetCredit(txout, filter); } CoinsList listCoins() override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); CoinsList result; - for (const auto& entry : m_wallet.ListCoins(*locked_chain)) { + for (const auto& entry : m_wallet->ListCoins(*locked_chain)) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*locked_chain, m_wallet, *coin.tx, coin.i, coin.nDepth)); + MakeWalletTxOut(*locked_chain, *m_wallet, *coin.tx, coin.i, coin.nDepth)); } } return result; } std::vector<WalletTxOut> getCoins(const std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); std::vector<WalletTxOut> result; result.reserve(outputs.size()); for (const auto& output : outputs) { result.emplace_back(); - auto it = m_wallet.mapWallet.find(output.hash); - if (it != m_wallet.mapWallet.end()) { + auto it = m_wallet->mapWallet.find(output.hash); + if (it != m_wallet->mapWallet.end()) { int depth = it->second.GetDepthInMainChain(*locked_chain); if (depth >= 0) { - result.back() = MakeWalletTxOut(*locked_chain, m_wallet, it->second, output.n, depth); + result.back() = MakeWalletTxOut(*locked_chain, *m_wallet, it->second, output.n, depth); } } } return result; } - CAmount getRequiredFee(unsigned int tx_bytes) override { return GetRequiredFee(m_wallet, tx_bytes); } + CAmount getRequiredFee(unsigned int tx_bytes) override { return GetRequiredFee(*m_wallet, tx_bytes); } CAmount getMinimumFee(unsigned int tx_bytes, const CCoinControl& coin_control, int* returned_target, @@ -457,55 +451,54 @@ public: { FeeCalculation fee_calc; CAmount result; - result = GetMinimumFee(m_wallet, tx_bytes, coin_control, ::mempool, ::feeEstimator, &fee_calc); + result = GetMinimumFee(*m_wallet, tx_bytes, coin_control, &fee_calc); if (returned_target) *returned_target = fee_calc.returnedTarget; if (reason) *reason = fee_calc.reason; return result; } - unsigned int getConfirmTarget() override { return m_wallet.m_confirm_target; } - bool hdEnabled() override { return m_wallet.IsHDEnabled(); } - bool canGetAddresses() override { return m_wallet.CanGetAddresses(); } - bool IsWalletFlagSet(uint64_t flag) override { return m_wallet.IsWalletFlagSet(flag); } - OutputType getDefaultAddressType() override { return m_wallet.m_default_address_type; } - OutputType getDefaultChangeType() override { return m_wallet.m_default_change_type; } + unsigned int getConfirmTarget() override { return m_wallet->m_confirm_target; } + bool hdEnabled() override { return m_wallet->IsHDEnabled(); } + bool canGetAddresses() override { return m_wallet->CanGetAddresses(); } + bool IsWalletFlagSet(uint64_t flag) override { return m_wallet->IsWalletFlagSet(flag); } + OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } + OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } void remove() override { - RemoveWallet(m_shared_wallet); + RemoveWallet(m_wallet); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { - return MakeHandler(m_wallet.NotifyUnload.connect(fn)); + return MakeHandler(m_wallet->NotifyUnload.connect(fn)); } std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override { - return MakeHandler(m_wallet.ShowProgress.connect(fn)); + return MakeHandler(m_wallet->ShowProgress.connect(fn)); } std::unique_ptr<Handler> handleStatusChanged(StatusChangedFn fn) override { - return MakeHandler(m_wallet.NotifyStatusChanged.connect([fn](CCryptoKeyStore*) { fn(); })); + return MakeHandler(m_wallet->NotifyStatusChanged.connect([fn](CCryptoKeyStore*) { fn(); })); } std::unique_ptr<Handler> handleAddressBookChanged(AddressBookChangedFn fn) override { - return MakeHandler(m_wallet.NotifyAddressBookChanged.connect( + return MakeHandler(m_wallet->NotifyAddressBookChanged.connect( [fn](CWallet*, const CTxDestination& address, const std::string& label, bool is_mine, const std::string& purpose, ChangeType status) { fn(address, label, is_mine, purpose, status); })); } std::unique_ptr<Handler> handleTransactionChanged(TransactionChangedFn fn) override { - return MakeHandler(m_wallet.NotifyTransactionChanged.connect( + return MakeHandler(m_wallet->NotifyTransactionChanged.connect( [fn](CWallet*, const uint256& txid, ChangeType status) { fn(txid, status); })); } std::unique_ptr<Handler> handleWatchOnlyChanged(WatchOnlyChangedFn fn) override { - return MakeHandler(m_wallet.NotifyWatchonlyChanged.connect(fn)); + return MakeHandler(m_wallet->NotifyWatchonlyChanged.connect(fn)); } std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) override { - return MakeHandler(m_wallet.NotifyCanGetAddressesChanged.connect(fn)); + return MakeHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); } - std::shared_ptr<CWallet> m_shared_wallet; - CWallet& m_wallet; + std::shared_ptr<CWallet> m_wallet; }; class WalletClientImpl : public ChainClient @@ -515,7 +508,7 @@ public: : m_chain(chain), m_wallet_filenames(std::move(wallet_filenames)) { } - void registerRpcs() override { return RegisterWalletRPCCommands(::tableRPC); } + void registerRpcs() override { return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); } bool verify() override { return VerifyWallets(m_chain, m_wallet_filenames); } bool load() override { return LoadWallets(m_chain, m_wallet_filenames); } void start(CScheduler& scheduler) override { return StartWallets(scheduler); } @@ -525,6 +518,7 @@ public: Chain& m_chain; std::vector<std::string> m_wallet_filenames; + std::vector<std::unique_ptr<Handler>> m_rpc_handlers; }; } // namespace diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index a931e5fafb..cabc455b1f 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -292,9 +292,6 @@ public: //! Get transaction data. virtual const CTransaction& get() = 0; - //! Get virtual transaction size. - virtual int64_t getVirtualSize() = 0; - //! Send pending transaction and commit to wallet. virtual bool commit(WalletValueMap value_map, WalletOrderForm order_form, diff --git a/src/net.cpp b/src/net.cpp index 87f1ef0577..1335804b06 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1765,9 +1765,15 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) addr = addrman.Select(fFeeler); } - // if we selected an invalid address, restart - if (!addr.IsValid() || setConnected.count(addr.GetGroup()) || IsLocal(addr)) + // Require outbound connections, other than feelers, to be to distinct network groups + if (!fFeeler && setConnected.count(addr.GetGroup())) { break; + } + + // if we selected an invalid or local address, restart + if (!addr.IsValid() || IsLocal(addr)) { + break; + } // If we didn't find an appropriate destination after trying 100 addresses fetched from addrman, // stop this loop, and let the outer loop run again (which sleeps, adds seed nodes, recalculates @@ -2621,7 +2627,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn { hSocket = hSocketIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; - strSubVer = ""; hashContinue = uint256(); filterInventoryKnown.reset(); pfilter = MakeUnique<CBloomFilter>(); @@ -53,7 +53,7 @@ static const unsigned int MAX_LOCATOR_SZ = 101; static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ static const unsigned int MAX_PROTOCOL_MESSAGE_LENGTH = 4 * 1000 * 1000; -/** Maximum length of strSubVer in `version` message */ +/** Maximum length of the user agent string in `version` message */ static const unsigned int MAX_SUBVERSION_LENGTH = 256; /** Maximum number of automatic outgoing nodes */ static const int MAX_OUTBOUND_CONNECTIONS = 8; @@ -650,12 +650,12 @@ public: // Bind address of our side of the connection const CAddress addrBind; std::atomic<int> nVersion{0}; - // strSubVer is whatever byte array we read from the wire. However, this field is intended - // to be printed out, displayed to humans in various forms and so on. So we sanitize it and - // store the sanitized version in cleanSubVer. The original should be used when dealing with - // the network or wire types and the cleaned string used when displayed or logged. - std::string strSubVer GUARDED_BY(cs_SubVer), cleanSubVer GUARDED_BY(cs_SubVer); - CCriticalSection cs_SubVer; // used for both cleanSubVer and strSubVer + RecursiveMutex cs_SubVer; + /** + * cleanSubVer is a sanitized string of the user agent byte array we read + * from the wire. This cleaned string can safely be logged or displayed. + */ + std::string cleanSubVer GUARDED_BY(cs_SubVer){}; bool m_prefer_evict{false}; // This peer is preferred for eviction. bool fWhitelisted{false}; // This peer can bypass DoS banning. bool fFeeler{false}; // If true this node is being used as a short lived feeler. @@ -739,6 +739,8 @@ public: CAmount lastSentFeeFilter{0}; int64_t nextSendTimeFeeFilter{0}; + std::set<uint256> orphan_work_set; + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); CNode(const CNode&) = delete; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 18dd2f010c..0f13d6e269 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1713,6 +1713,67 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve return true; } +void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +{ + AssertLockHeld(cs_main); + AssertLockHeld(g_cs_orphans); + std::set<NodeId> setMisbehaving; + bool done = false; + while (!done && !orphan_work_set.empty()) { + const uint256 orphanHash = *orphan_work_set.begin(); + orphan_work_set.erase(orphan_work_set.begin()); + + auto orphan_it = mapOrphanTransactions.find(orphanHash); + if (orphan_it == mapOrphanTransactions.end()) continue; + + const CTransactionRef porphanTx = orphan_it->second.tx; + const CTransaction& orphanTx = *porphanTx; + NodeId fromPeer = orphan_it->second.fromPeer; + bool fMissingInputs2 = false; + // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan + // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get + // anyone relaying LegitTxX banned) + CValidationState stateDummy; + + if (setMisbehaving.count(fromPeer)) continue; + if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, &fMissingInputs2, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); + RelayTransaction(orphanTx, connman); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + orphan_work_set.insert(elem->first); + } + } + } + EraseOrphanTx(orphanHash); + done = true; + } else if (!fMissingInputs2) { + int nDos = 0; + if (stateDummy.IsInvalid(nDos) && nDos > 0) { + // Punish peer that gave us an invalid orphan tx + Misbehaving(fromPeer, nDos); + setMisbehaving.insert(fromPeer); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); + } + // Has inputs but not accepted to mempool + // Probably non-standard or insufficient fee + LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); + if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + assert(recentRejects); + recentRejects->insert(orphanHash); + } + EraseOrphanTx(orphanHash); + done = true; + } + mempool.check(pcoinsTip.get()); + } +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc, bool enable_bip61) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -1782,7 +1843,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ServiceFlags nServices; int nVersion; int nSendVersion; - std::string strSubVer; std::string cleanSubVer; int nStartingHeight = -1; bool fRelay = true; @@ -1819,6 +1879,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!vRecv.empty()) vRecv >> addrFrom >> nNonce; if (!vRecv.empty()) { + std::string strSubVer; vRecv >> LIMITED_STRING(strSubVer, MAX_SUBVERSION_LENGTH); cleanSubVer = SanitizeString(strSubVer); } @@ -1850,7 +1911,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->SetAddrLocal(addrMe); { LOCK(pfrom->cs_SubVer); - pfrom->strSubVer = strSubVer; pfrom->cleanSubVer = cleanSubVer; } pfrom->nStartingHeight = nStartingHeight; @@ -2022,6 +2082,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); + if (g_banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -2341,8 +2402,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - std::deque<COutPoint> vWorkQueue; - std::vector<uint256> vEraseQueue; CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; @@ -2367,7 +2426,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.check(pcoinsTip.get()); RelayTransaction(tx, connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { - vWorkQueue.emplace_back(inv.hash, i); + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); + if (it_by_prev != mapOrphanTransactionsByPrev.end()) { + for (const auto& elem : it_by_prev->second) { + pfrom->orphan_work_set.insert(elem->first); + } + } } pfrom->nLastTXTime = GetTime(); @@ -2378,65 +2442,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - std::set<NodeId> setMisbehaving; - while (!vWorkQueue.empty()) { - auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); - vWorkQueue.pop_front(); - if (itByPrev == mapOrphanTransactionsByPrev.end()) - continue; - for (auto mi = itByPrev->second.begin(); - mi != itByPrev->second.end(); - ++mi) - { - const CTransactionRef& porphanTx = (*mi)->second.tx; - const CTransaction& orphanTx = *porphanTx; - const uint256& orphanHash = orphanTx.GetHash(); - NodeId fromPeer = (*mi)->second.fromPeer; - bool fMissingInputs2 = false; - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan - // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get - // anyone relaying LegitTxX banned) - CValidationState stateDummy; - - - if (setMisbehaving.count(fromPeer)) - continue; - if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, &fMissingInputs2, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, connman); - for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { - vWorkQueue.emplace_back(orphanHash, i); - } - vEraseQueue.push_back(orphanHash); - } - else if (!fMissingInputs2) - { - int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) - { - // Punish peer that gave us an invalid orphan tx - Misbehaving(fromPeer, nDos); - setMisbehaving.insert(fromPeer); - LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); - } - // Has inputs but not accepted to mempool - // Probably non-standard or insufficient fee - LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - vEraseQueue.push_back(orphanHash); - if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. - assert(recentRejects); - recentRejects->insert(orphanHash); - } - } - mempool.check(pcoinsTip.get()); - } - } - - for (const uint256& hash : vEraseQueue) - EraseOrphanTx(hash); + ProcessOrphanTx(connman, pfrom->orphan_work_set, lRemovedTxn); } else if (fMissingInputs) { @@ -2540,8 +2546,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::CMPCTBLOCK) { + // Ignore cmpctblock received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected cmpctblock message received from peer %d\n", pfrom->GetId()); + return true; + } + CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; @@ -2761,8 +2773,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::BLOCKTXN) { + // Ignore blocktxn received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected blocktxn message received from peer %d\n", pfrom->GetId()); + return true; + } + BlockTransactions resp; vRecv >> resp; @@ -2836,8 +2854,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } - if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing + if (strCommand == NetMsgType::HEADERS) { + // Ignore headers received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom->GetId()); + return true; + } + std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -2861,8 +2885,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); } - if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + if (strCommand == NetMsgType::BLOCK) { + // Ignore block received while importing + if (fImporting || fReindex) { + LogPrint(BCLog::NET, "Unexpected block message received from peer %d\n", pfrom->GetId()); + return true; + } + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; @@ -2912,8 +2942,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->vAddrToSend.clear(); std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; - for (const CAddress &addr : vAddr) - pfrom->PushAddress(addr, insecure_rand); + for (const CAddress &addr : vAddr) { + if (!g_banman->IsBanned(addr)) { + pfrom->PushAddress(addr, insecure_rand); + } + } return true; } @@ -3141,11 +3174,21 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!pfrom->vRecvGetData.empty()) ProcessGetData(pfrom, chainparams, connman, interruptMsgProc); + if (!pfrom->orphan_work_set.empty()) { + std::list<CTransactionRef> removed_txn; + LOCK2(cs_main, g_cs_orphans); + ProcessOrphanTx(connman, pfrom->orphan_work_set, removed_txn); + for (const CTransactionRef& removedTx : removed_txn) { + AddToCompactExtraTransactions(removedTx); + } + } + if (pfrom->fDisconnect) return false; // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; + if (!pfrom->orphan_work_set.empty()) return true; // Don't bother if send buffer is too full to respond anyway if (pfrom->fPauseSend) diff --git a/src/node/coin.cpp b/src/node/coin.cpp new file mode 100644 index 0000000000..bb98e63f3a --- /dev/null +++ b/src/node/coin.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/coin.h> + +#include <txmempool.h> +#include <validation.h> + +void FindCoins(std::map<COutPoint, Coin>& coins) +{ + LOCK2(cs_main, ::mempool.cs); + assert(pcoinsTip); + CCoinsViewCache& chain_view = *::pcoinsTip; + CCoinsViewMemPool mempool_view(&chain_view, ::mempool); + for (auto& coin : coins) { + if (!mempool_view.GetCoin(coin.first, coin.second)) { + // Either the coin is not in the CCoinsViewCache or is spent. Clear it. + coin.second.Clear(); + } + } +} diff --git a/src/node/coin.h b/src/node/coin.h new file mode 100644 index 0000000000..eb95b75cfb --- /dev/null +++ b/src/node/coin.h @@ -0,0 +1,22 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_COIN_H +#define BITCOIN_NODE_COIN_H + +#include <map> + +class COutPoint; +class Coin; + +/** + * Look up unspent output information. Returns coins in the mempool and in the + * current chain UTXO set. Iterates through all the keys in the map and + * populates the values. + * + * @param[in,out] coins map to fill + */ +void FindCoins(std::map<COutPoint, Coin>& coins); + +#endif // BITCOIN_NODE_COIN_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index c9cdd0d1cd..7b9b4310e7 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -12,7 +12,7 @@ #include <future> -const char* TransactionErrorString(const TransactionError err) +std::string TransactionErrorString(const TransactionError err) { switch (err) { case TransactionError::OK: @@ -33,22 +33,16 @@ const char* TransactionErrorString(const TransactionError err) return "PSBTs not compatible (different transactions)"; case TransactionError::SIGHASH_MISMATCH: return "Specified sighash value does not match existing value"; - - case TransactionError::UNKNOWN_ERROR: - default: break; + // no default case, so the compiler can warn about missing cases } - return "Unknown error"; + assert(false); } -bool BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, TransactionError& error, std::string& err_string, const bool allowhighfees) +TransactionError BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, std::string& err_string, const CAmount& highfee) { std::promise<void> promise; hashTx = tx->GetHash(); - CAmount nMaxRawTxFee = maxTxFee; - if (allowhighfees) - nMaxRawTxFee = 0; - { // cs_main scope LOCK(cs_main); CCoinsViewCache &view = *pcoinsTip; @@ -63,19 +57,16 @@ bool BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, Transaction CValidationState state; bool fMissingInputs; if (!AcceptToMemoryPool(mempool, state, std::move(tx), &fMissingInputs, - nullptr /* plTxnReplaced */, false /* bypass_limits */, nMaxRawTxFee)) { + nullptr /* plTxnReplaced */, false /* bypass_limits */, highfee)) { if (state.IsInvalid()) { err_string = FormatStateMessage(state); - error = TransactionError::MEMPOOL_REJECTED; - return false; + return TransactionError::MEMPOOL_REJECTED; } else { if (fMissingInputs) { - error = TransactionError::MISSING_INPUTS; - return false; + return TransactionError::MISSING_INPUTS; } err_string = FormatStateMessage(state); - error = TransactionError::MEMPOOL_ERROR; - return false; + return TransactionError::MEMPOOL_ERROR; } } else { // If wallet is enabled, ensure that the wallet has been made aware @@ -88,8 +79,7 @@ bool BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, Transaction }); } } else if (fHaveChain) { - error = TransactionError::ALREADY_IN_CHAIN; - return false; + return TransactionError::ALREADY_IN_CHAIN; } else { // Make sure we don't block forever if re-sending // a transaction already in mempool. @@ -100,16 +90,14 @@ bool BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, Transaction promise.get_future().wait(); - if(!g_connman) { - error = TransactionError::P2P_DISABLED; - return false; + if (!g_connman) { + return TransactionError::P2P_DISABLED; } CInv inv(MSG_TX, hashTx); - g_connman->ForEachNode([&inv](CNode* pnode) - { + g_connman->ForEachNode([&inv](CNode* pnode) { pnode->PushInventory(inv); }); - return true; - } + return TransactionError::OK; +} diff --git a/src/node/transaction.h b/src/node/transaction.h index 3b0cbba98b..3457ececa4 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -1,17 +1,16 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NODE_TRANSACTION_H #define BITCOIN_NODE_TRANSACTION_H +#include <attributes.h> #include <primitives/transaction.h> #include <uint256.h> enum class TransactionError { - OK = 0, - UNKNOWN_ERROR, - + OK, //!< No error MISSING_INPUTS, ALREADY_IN_CHAIN, P2P_DISABLED, @@ -20,24 +19,19 @@ enum class TransactionError { INVALID_PSBT, PSBT_MISMATCH, SIGHASH_MISMATCH, - - ERROR_COUNT }; -#define TRANSACTION_ERR_LAST TransactionError::ERROR_COUNT - -const char* TransactionErrorString(const TransactionError error); +std::string TransactionErrorString(const TransactionError error); /** * Broadcast a transaction * * @param[in] tx the transaction to broadcast * @param[out] &txid the txid of the transaction, if successfully broadcast - * @param[out] &error reference to UniValue to fill with error info on failure * @param[out] &err_string reference to std::string to fill with error string if available - * @param[in] allowhighfees whether to allow fees exceeding maxTxFee - * return true on success, false on error (and fills in `error`) + * @param[in] highfee Reject txs with fees higher than this (if 0, accept any fee) + * return error */ -bool BroadcastTransaction(CTransactionRef tx, uint256& txid, TransactionError& error, std::string& err_string, bool allowhighfees = false); +NODISCARD TransactionError BroadcastTransaction(CTransactionRef tx, uint256& txid, std::string& err_string, const CAmount& highfee); #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 0dc130d104..c73a97fd7d 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -14,7 +14,7 @@ bool SignalsOptInRBF(const CTransaction &tx) return false; } -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 581f489e12..a4f8777310 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -23,6 +23,6 @@ bool SignalsOptInRBF(const CTransaction &tx); // according to BIP 125 // This involves checking sequence numbers of the transaction, as well // as the sequence numbers of all in-mempool ancestors. -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/psbt.cpp b/src/psbt.cpp index 32fb459dec..184129e330 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -2,9 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> +#include <consensus/tx_verify.h> +#include <policy/policy.h> #include <psbt.h> #include <util/strencodings.h> +#include <numeric> + PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) { inputs.resize(tx.vin.size()); @@ -205,7 +210,7 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; } -bool PSBTInputSigned(PSBTInput& input) +bool PSBTInputSigned(const PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } @@ -309,21 +314,178 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti return true; } -bool CombinePSBTs(PartiallySignedTransaction& out, TransactionError& error, const std::vector<PartiallySignedTransaction>& psbtxs) +TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) { out = psbtxs[0]; // Copy the first one // Merge for (auto it = std::next(psbtxs.begin()); it != psbtxs.end(); ++it) { if (!out.Merge(*it)) { - error = TransactionError::PSBT_MISMATCH; - return false; + return TransactionError::PSBT_MISMATCH; } } if (!out.IsSane()) { - error = TransactionError::INVALID_PSBT; + return TransactionError::INVALID_PSBT; + } + + return TransactionError::OK; +} + +std::string PSBTRoleName(PSBTRole role) { + switch (role) { + case PSBTRole::UPDATER: return "updater"; + case PSBTRole::SIGNER: return "signer"; + case PSBTRole::FINALIZER: return "finalizer"; + case PSBTRole::EXTRACTOR: return "extractor"; + } +} + +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) +{ + // Go through each input and build status + PSBTAnalysis result; + + bool calc_fee = true; + bool all_final = true; + bool only_missing_sigs = true; + bool only_missing_final = false; + CAmount in_amt = 0; + + result.inputs.resize(psbtx.tx->vin.size()); + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + PSBTInputAnalysis& input_analysis = result.inputs[i]; + + // Check for a UTXO + CTxOut utxo; + if (psbtx.GetInputUTXO(utxo, i)) { + in_amt += utxo.nValue; + input_analysis.has_utxo = true; + } else { + input_analysis.has_utxo = false; + input_analysis.is_final = false; + input_analysis.next = PSBTRole::UPDATER; + calc_fee = false; + } + + // Check if it is final + if (!utxo.IsNull() && !PSBTInputSigned(input)) { + input_analysis.is_final = false; + all_final = false; + + // Figure out what is missing + SignatureData outdata; + bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); + + // Things are missing + if (!complete) { + input_analysis.missing_pubkeys = outdata.missing_pubkeys; + input_analysis.missing_redeem_script = outdata.missing_redeem_script; + input_analysis.missing_witness_script = outdata.missing_witness_script; + input_analysis.missing_sigs = outdata.missing_sigs; + + // If we are only missing signatures and nothing else, then next is signer + if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { + input_analysis.next = PSBTRole::SIGNER; + } else { + only_missing_sigs = false; + input_analysis.next = PSBTRole::UPDATER; + } + } else { + only_missing_final = true; + input_analysis.next = PSBTRole::FINALIZER; + } + } else if (!utxo.IsNull()){ + input_analysis.is_final = true; + } + } + + if (all_final) { + only_missing_sigs = false; + result.next = PSBTRole::EXTRACTOR; + } + if (calc_fee) { + // Get the output amount + CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), + [](CAmount a, const CTxOut& b) { + return a += b.nValue; + } + ); + + // Get the fee + CAmount fee = in_amt - out_amt; + result.fee = fee; + + // Estimate the size + CMutableTransaction mtx(*psbtx.tx); + CCoinsView view_dummy; + CCoinsViewCache view(&view_dummy); + bool success = true; + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + Coin newcoin; + + if (!SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true) || !psbtx.GetInputUTXO(newcoin.out, i)) { + success = false; + break; + } else { + mtx.vin[i].scriptSig = input.final_script_sig; + mtx.vin[i].scriptWitness = input.final_script_witness; + newcoin.nHeight = 1; + view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); + } + } + + if (success) { + CTransaction ctx = CTransaction(mtx); + size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); + result.estimated_vsize = size; + // Estimate fee rate + CFeeRate feerate(fee, size); + result.estimated_feerate = feerate; + } + + if (only_missing_sigs) { + result.next = PSBTRole::SIGNER; + } else if (only_missing_final) { + result.next = PSBTRole::FINALIZER; + } else if (all_final) { + result.next = PSBTRole::EXTRACTOR; + } else { + result.next = PSBTRole::UPDATER; + } + } else { + result.next = PSBTRole::UPDATER; + } + + return result; +} + +bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) +{ + bool invalid; + std::string tx_data = DecodeBase64(base64_tx, &invalid); + if (invalid) { + error = "invalid base64"; return false; } + return DecodeRawPSBT(psbt, tx_data, error); +} +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +{ + CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + try { + ss_data >> psbt; + if (!ss_data.empty()) { + error = "extra data after PSBT"; + return false; + } + } catch (const std::exception& e) { + error = e.what(); + return false; + } return true; } diff --git a/src/psbt.h b/src/psbt.h index 27b0aedd05..fcb3337a53 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,8 @@ #include <attributes.h> #include <node/transaction.h> +#include <optional.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <pubkey.h> #include <script/sign.h> @@ -548,8 +550,42 @@ struct PartiallySignedTransaction } }; +enum class PSBTRole { + UPDATER, + SIGNER, + FINALIZER, + EXTRACTOR +}; + +/** + * Holds an analysis of one input from a PSBT + */ +struct PSBTInputAnalysis { + bool has_utxo; //!< Whether we have UTXO information for this input + bool is_final; //!< Whether the input has all required information including signatures + PSBTRole next; //!< Which of the BIP 174 roles needs to handle this input next + + std::vector<CKeyID> missing_pubkeys; //!< Pubkeys whose BIP32 derivation path is missing + std::vector<CKeyID> missing_sigs; //!< Pubkeys whose signatures are missing + uint160 missing_redeem_script; //!< Hash160 of redeem script, if missing + uint256 missing_witness_script; //!< SHA256 of witness script, if missing +}; + +/** + * Holds the results of AnalyzePSBT (miscellaneous information about a PSBT) + */ +struct PSBTAnalysis { + Optional<size_t> estimated_vsize; //!< Estimated weight of the transaction + Optional<CFeeRate> estimated_feerate; //!< Estimated feerate (fee / weight) of the transaction + Optional<CAmount> fee; //!< Amount of fee being paid by the transaction + std::vector<PSBTInputAnalysis> inputs; //!< More information about the individual inputs of the transaction + PSBTRole next; //!< Which of the BIP 174 roles needs to handle the transaction next +}; + +std::string PSBTRoleName(PSBTRole role); + /** Checks whether a PSBTInput is already signed. */ -bool PSBTInputSigned(PSBTInput& input); +bool PSBTInputSigned(const PSBTInput& input); /** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); @@ -575,10 +611,22 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti * Combines PSBTs with the same underlying transaction, resulting in a single PSBT with all partial signatures from each input. * * @param[out] &out the combined PSBT, if successful - * @param[out] &error reference to TransactionError to fill with error info on failure * @param[in] psbtxs the PSBTs to combine - * @return True if we successfully combined the transactions, false if they were not compatible + * @return error (OK if we successfully combined the transactions, other error if they were not compatible) + */ +NODISCARD TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); + +/** + * Provides helpful miscellaneous information about where a PSBT is in the signing workflow. + * + * @param[in] psbtx the PSBT to analyze + * @return A PSBTAnalysis with information about the provided PSBT. */ -bool CombinePSBTs(PartiallySignedTransaction& out, TransactionError& error, const std::vector<PartiallySignedTransaction>& psbtxs); +PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); + +//! Decode a base64ed PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); +//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); #endif // BITCOIN_PSBT_H diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index bc88dd5e0a..abf9136eee 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -396,6 +396,8 @@ void BitcoinGUI::createActions() connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); connect(activity, &OpenWalletActivity::finished, activity, &QObject::deleteLater); connect(activity, &OpenWalletActivity::finished, dialog, &QObject::deleteLater); + bool invoked = QMetaObject::invokeMethod(activity, "open"); + assert(invoked); }); } }); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index fc1e14b031..c7ced3c106 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -69,7 +69,6 @@ const QStringList historyFilter = QStringList() << "importmulti" << "sethdseed" << "signmessagewithprivkey" - << "signrawtransaction" << "signrawtransactionwithkey" << "walletpassphrase" << "walletpassphrasechange" diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index c532ffbbfe..019bd65823 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -9,9 +9,11 @@ #include <algorithm> +#include <QApplication> #include <QMessageBox> #include <QMutexLocker> #include <QThread> +#include <QWindow> WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) : QObject(parent) @@ -59,7 +61,6 @@ OpenWalletActivity* WalletController::openWallet(const std::string& name, QWidge { OpenWalletActivity* activity = new OpenWalletActivity(this, name); activity->moveToThread(&m_activity_thread); - QMetaObject::invokeMethod(activity, "open", Qt::QueuedConnection); return activity; } @@ -98,7 +99,17 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal m_wallets.push_back(wallet_model); connect(wallet_model, &WalletModel::unload, [this, wallet_model] { - removeAndDeleteWallet(wallet_model); + // Defer removeAndDeleteWallet when no modal widget is active. + // TODO: remove this workaround by removing usage of QDiallog::exec. + if (QApplication::activeModalWidget()) { + connect(qApp, &QApplication::focusWindowChanged, wallet_model, [this, wallet_model]() { + if (!QApplication::activeModalWidget()) { + removeAndDeleteWallet(wallet_model); + } + }, Qt::QueuedConnection); + } else { + removeAndDeleteWallet(wallet_model); + } }); // Re-emit coinsSent signal from wallet model. diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 466f2278eb..94413547d4 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -40,15 +40,11 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) this->clientModel = _clientModel; } -bool WalletFrame::addWallet(WalletModel *walletModel) +void WalletFrame::addWallet(WalletModel *walletModel) { - if (!gui || !clientModel || !walletModel) { - return false; - } + if (!gui || !clientModel || !walletModel) return; - if (mapWalletViews.count(walletModel) > 0) { - return false; - } + if (mapWalletViews.count(walletModel) > 0) return; WalletView *walletView = new WalletView(platformStyle, this); walletView->setBitcoinGUI(gui); @@ -72,31 +68,25 @@ bool WalletFrame::addWallet(WalletModel *walletModel) }); connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); - - return true; } -bool WalletFrame::setCurrentWallet(WalletModel* wallet_model) +void WalletFrame::setCurrentWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.value(wallet_model); walletStack->setCurrentWidget(walletView); assert(walletView); walletView->updateEncryptionStatus(); - return true; } -bool WalletFrame::removeWallet(WalletModel* wallet_model) +void WalletFrame::removeWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.take(wallet_model); walletStack->removeWidget(walletView); delete walletView; - return true; } void WalletFrame::removeAllWallets() diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 6a74fde9fd..156653f47d 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -36,9 +36,9 @@ public: void setClientModel(ClientModel *clientModel); - bool addWallet(WalletModel *walletModel); - bool setCurrentWallet(WalletModel* wallet_model); - bool removeWallet(WalletModel* wallet_model); + void addWallet(WalletModel *walletModel); + void setCurrentWallet(WalletModel* wallet_model); + void removeWallet(WalletModel* wallet_model); void removeAllWallets(); bool handlePaymentRequest(const SendCoinsRecipient& recipient); diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eb3b0baf08..2694d67800 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -29,7 +29,7 @@ std::unique_ptr<interfaces::PendingWalletTx>& WalletModelTransaction::getWtx() unsigned int WalletModelTransaction::getTransactionSize() { - return wtx ? wtx->getVirtualSize() : 0; + return wtx ? GetVirtualTransactionSize(wtx->get()) : 0; } CAmount WalletModelTransaction::getTransactionFee() const diff --git a/src/rest.cpp b/src/rest.cpp index 326f7ae1d2..baad3b2ce9 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -300,7 +300,7 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) switch (rf) { case RetFormat::JSON: { - UniValue mempoolInfoObject = mempoolInfoToJSON(); + UniValue mempoolInfoObject = MempoolInfoToJSON(::mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -322,7 +322,7 @@ static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPar switch (rf) { case RetFormat::JSON: { - UniValue mempoolObject = mempoolToJSON(true); + UniValue mempoolObject = MempoolToJSON(::mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7fb9ff2eaf..d35f458b2e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -378,7 +378,9 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::string EntryDescriptionString() { - return " \"size\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + return " \"vsize\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" + " \"size\" : n, (numeric) (DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" + " size will be completely removed in v0.20.\n" " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)\n" " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority (DEPRECATED)\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" @@ -405,9 +407,9 @@ static std::string EntryDescriptionString() " \"bip125-replaceable\" : true|false, (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee)\n"; } -static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCKS_REQUIRED(::mempool.cs) +static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { - AssertLockHeld(mempool.cs); + AssertLockHeld(pool.cs); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -416,7 +418,8 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK fees.pushKV("descendant", ValueFromAmount(e.GetModFeesWithDescendants())); info.pushKV("fees", fees); - info.pushKV("size", (int)e.GetTxSize()); + info.pushKV("vsize", (int)e.GetTxSize()); + if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); info.pushKV("time", e.GetTime()); @@ -427,12 +430,12 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); - info.pushKV("wtxid", mempool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); const CTransaction& tx = e.GetTx(); std::set<std::string> setDepends; for (const CTxIn& txin : tx.vin) { - if (mempool.exists(txin.prevout.hash)) + if (pool.exists(txin.prevout.hash)) setDepends.insert(txin.prevout.hash.ToString()); } @@ -445,8 +448,8 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter &it = mempool.mapTx.find(tx.GetHash()); - const CTxMemPool::setEntries &setChildren = mempool.GetMemPoolChildren(it); + const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); + const CTxMemPool::setEntries& setChildren = pool.GetMemPoolChildren(it); for (CTxMemPool::txiter childiter : setChildren) { spent.push_back(childiter->GetTx().GetHash().ToString()); } @@ -455,7 +458,7 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK // Add opt-in RBF status bool rbfStatus = false; - RBFTransactionState rbfState = IsRBFOptIn(tx, mempool); + RBFTransactionState rbfState = IsRBFOptIn(tx, pool); if (rbfState == RBFTransactionState::UNKNOWN) { throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { @@ -465,25 +468,21 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("bip125-replaceable", rbfStatus); } -UniValue mempoolToJSON(bool fVerbose) +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose) { - if (fVerbose) - { - LOCK(mempool.cs); + if (verbose) { + LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : mempool.mapTx) - { + for (const CTxMemPoolEntry& e : pool.mapTx) { const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(pool, info, e); o.pushKV(hash.ToString(), info); } return o; - } - else - { + } else { std::vector<uint256> vtxid; - mempool.queryHashes(vtxid); + pool.queryHashes(vtxid); UniValue a(UniValue::VARR); for (const uint256& hash : vtxid) @@ -525,7 +524,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return mempoolToJSON(fVerbose); + return MempoolToJSON(::mempool, fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -591,7 +590,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *ancestorIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -661,7 +660,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *descendantIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -700,7 +699,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); return info; } @@ -1485,15 +1484,15 @@ static UniValue getchaintips(const JSONRPCRequest& request) return res; } -UniValue mempoolInfoToJSON() +UniValue MempoolInfoToJSON(const CTxMemPool& pool) { UniValue ret(UniValue::VOBJ); - ret.pushKV("size", (int64_t) mempool.size()); - ret.pushKV("bytes", (int64_t) mempool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t) mempool.DynamicMemoryUsage()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(std::max(mempool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); + ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); return ret; @@ -1522,7 +1521,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.ToString()); - return mempoolInfoToJSON(); + return MempoolInfoToJSON(::mempool); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1583,15 +1582,15 @@ static UniValue invalidateblock(const JSONRPCRequest& request) uint256 hash(ParseHashV(request.params[0], "blockhash")); CValidationState state; + CBlockIndex* pblockindex; { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + pblockindex = LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - - InvalidateBlock(state, Params(), pblockindex); } + InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { ActivateBestChain(state, Params()); @@ -1778,9 +1777,7 @@ static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) static UniValue getblockstats(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) { - throw std::runtime_error( - RPCHelpMan{"getblockstats", + const RPCHelpMan help{"getblockstats", "\nCompute per block statistics for a given window. All amounts are in satoshis.\n" "It won't work for some heights with pruning.\n" "It won't work without -txindex for utxo_size_inc, *fee or *feerate stats.\n", @@ -1836,7 +1833,9 @@ static UniValue getblockstats(const JSONRPCRequest& request) HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") }, - }.ToString()); + }; + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); } LOCK(cs_main); @@ -2159,7 +2158,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", { {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, - {"range", RPCArg::Type::NUM, /* default */ "1000", "Up to what child index HD chains should be explored"}, + {"range", RPCArg::Type::RANGE, /* default */ "1000", "The range of HD chain indexes to explore (either end or [begin,end])"}, }, }, }, @@ -2216,7 +2215,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) // loop through the scan objects for (const UniValue& scanobject : request.params[1].get_array().getValues()) { std::string desc_str; - int range = 1000; + std::pair<int64_t, int64_t> range = {0, 1000}; if (scanobject.isStr()) { desc_str = scanobject.get_str(); } else if (scanobject.isObject()) { @@ -2225,8 +2224,8 @@ UniValue scantxoutset(const JSONRPCRequest& request) desc_str = desc_uni.get_str(); UniValue range_uni = find_value(scanobject, "range"); if (!range_uni.isNull()) { - range = range_uni.get_int(); - if (range < 0 || range > 1000000) throw JSONRPCError(RPC_INVALID_PARAMETER, "range out of range"); + range = ParseRange(range_uni); + if (range.first < 0 || (range.second >> 31) != 0 || range.second >= range.first + 1000000) throw JSONRPCError(RPC_INVALID_PARAMETER, "range out of range"); } } else { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan object needs to be either a string or an object"); @@ -2237,8 +2236,11 @@ UniValue scantxoutset(const JSONRPCRequest& request) if (!desc) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor '%s'", desc_str)); } - if (!desc->IsRange()) range = 0; - for (int i = 0; i <= range; ++i) { + if (!desc->IsRange()) { + range.first = 0; + range.second = 0; + } + for (int i = range.first; i <= range.second; ++i) { std::vector<CScript> scripts; if (!desc->Expand(i, provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys: '%s'", desc_str)); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 529132d033..55d1de453f 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -11,6 +11,7 @@ class CBlock; class CBlockIndex; +class CTxMemPool; class UniValue; static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -30,10 +31,10 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false); /** Mempool information to JSON */ -UniValue mempoolInfoToJSON(); +UniValue MempoolInfoToJSON(const CTxMemPool& pool); /** Mempool to JSON */ -UniValue mempoolToJSON(bool fVerbose = false); +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false); /** Block header to JSON */ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 1cdc9f87a7..4144a17bc3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -28,8 +28,6 @@ public: static const CRPCConvertParam vRPCConvertParams[] = { { "setmocktime", 0, "timestamp" }, - { "generate", 0, "nblocks" }, - { "generate", 1, "maxtries" }, { "generatetoaddress", 0, "nblocks" }, { "generatetoaddress", 2, "maxtries" }, { "getnetworkhashps", 0, "nblocks" }, @@ -68,8 +66,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 4, "subtractfeefrom" }, { "sendmany", 5 , "replaceable" }, { "sendmany", 6 , "conf_target" }, - { "deriveaddresses", 1, "begin" }, - { "deriveaddresses", 2, "end" }, + { "deriveaddresses", 1, "range" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -95,8 +92,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, { "sendrawtransaction", 1, "allowhighfees" }, + { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "allowhighfees" }, + { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 2, "iswitness" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 6625a03bbd..480d610235 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -444,10 +444,10 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Bitcoin is not connected!"); + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Bitcoin is downloading blocks..."); + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; @@ -843,7 +843,8 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); bool conservative = true; if (!request.params[1].isNull()) { FeeEstimateMode fee_mode; @@ -915,7 +916,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); double threshold = 0.95; if (!request.params[1].isNull()) { threshold = request.params[1].get_real(); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 999a307e2b..0a97f80297 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -34,11 +34,7 @@ static UniValue validateaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( RPCHelpMan{"validateaddress", - "\nReturn information about the given bitcoin address.\n" - "DEPRECATION WARNING: Parts of this command have been deprecated and moved to getaddressinfo. Clients must\n" - "transition to using getaddressinfo to access this information before upgrading to v0.18. The following deprecated\n" - "fields have moved to getaddressinfo and will only be shown here with -deprecatedrpc=validateaddress: ismine, iswatchonly,\n" - "script, hex, pubkeys, sigsrequired, pubkey, addresses, embedded, iscompressed, account, timestamp, hdkeypath, kdmasterkeyid.\n", + "\nReturn information about the given bitcoin address.\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, }, @@ -185,7 +181,7 @@ UniValue getdescriptorinfo(const JSONRPCRequest& request) UniValue deriveaddresses(const JSONRPCRequest& request) { - if (request.fHelp || request.params.empty() || request.params.size() > 3) { + if (request.fHelp || request.params.empty() || request.params.size() > 2) { throw std::runtime_error( RPCHelpMan{"deriveaddresses", {"\nDerives one or more addresses corresponding to an output descriptor.\n" @@ -199,37 +195,37 @@ UniValue deriveaddresses(const JSONRPCRequest& request) "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, { {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, - {"begin", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the beginning of the range to import."}, - {"end", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end of the range to import."} + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, }, RPCResult{ "[ address ] (array) the derived addresses\n" }, RPCExamples{ "First three native segwit receive addresses\n" + - HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#trd0mf0l\" 0 2") + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#trd0mf0l\" \"[0,2]\"") }}.ToString() ); } - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VNUM, UniValue::VNUM}); + RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later const std::string desc_str = request.params[0].get_str(); - int range_begin = 0; - int range_end = 0; + int64_t range_begin = 0; + int64_t range_end = 0; - if (request.params.size() >= 2) { - if (request.params.size() == 2) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing range end parameter"); - } - range_begin = request.params[1].get_int(); - range_end = request.params[2].get_int(); - if (range_begin < 0) { + if (request.params.size() >= 2 && !request.params[1].isNull()) { + auto range = ParseRange(request.params[1]); + if (range.first < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should be greater or equal than 0"); } - if (range_begin > range_end) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Range end should be equal to or greater than begin"); + if ((range.second >> 31) != 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "End of range is too high"); + } + if (range.second >= range.first + 1000000) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range is too large"); } + range_begin = range.first; + range_end = range.second; } FlatSigningProvider provider; @@ -603,7 +599,7 @@ static const CRPCCommand commands[] = { "control", "logging", &logging, {"include", "exclude"}}, { "util", "validateaddress", &validateaddress, {"address"} }, { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, - { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "begin", "end"} }, + { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "range"} }, { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 6bbbbc9876..c7b3478f44 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -523,13 +523,7 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) static UniValue setban(const JSONRPCRequest& request) { - std::string strCommand; - if (!request.params[1].isNull()) - strCommand = request.params[1].get_str(); - if (request.fHelp || request.params.size() < 2 || - (strCommand != "add" && strCommand != "remove")) - throw std::runtime_error( - RPCHelpMan{"setban", + const RPCHelpMan help{"setban", "\nAttempts to add or remove an IP/Subnet from the banned list.\n", { {"subnet", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP/Subnet (see getpeerinfo for nodes IP) with an optional netmask (default is /32 = single IP)"}, @@ -543,7 +537,13 @@ static UniValue setban(const JSONRPCRequest& request) + HelpExampleCli("setban", "\"192.168.0.0/24\" \"add\"") + HelpExampleRpc("setban", "\"192.168.0.6\", \"add\", 86400") }, - }.ToString()); + }; + std::string strCommand; + if (!request.params[1].isNull()) + strCommand = request.params[1].get_str(); + if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { + throw std::runtime_error(help.ToString()); + } if (!g_banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 38e2dc237e..8d9c207011 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -6,11 +6,12 @@ #include <chain.h> #include <coins.h> #include <compat/byteswap.h> -#include <consensus/validation.h> #include <consensus/tx_verify.h> +#include <consensus/validation.h> #include <core_io.h> #include <index/txindex.h> #include <init.h> +#include <interfaces/chain.h> #include <key_io.h> #include <keystore.h> #include <merkleblock.h> @@ -28,6 +29,7 @@ #include <script/standard.h> #include <uint256.h> #include <util/bip32.h> +#include <util/moneystr.h> #include <util/strencodings.h> #include <validation.h> #include <validationinterface.h> @@ -38,6 +40,11 @@ #include <univalue.h> +/** High fee for sendrawtransaction and testmempoolaccept. + * By default, transaction with a fee higher than this will be rejected by the + * RPCs. This can be overriden with the maxfeerate argument. + */ +constexpr static CAmount DEFAULT_MAX_RAW_TX_FEE{COIN / 10}; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { @@ -67,9 +74,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& static UniValue getrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) - throw std::runtime_error( - RPCHelpMan{ + const RPCHelpMan help{ "getrawtransaction", "\nReturn the raw transaction data.\n" @@ -79,8 +84,7 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) "will return the transaction if it is in the mempool, or if -txindex is enabled and the transaction\n" "is in a block in the blockchain.\n" - "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" - "Or use gettransaction for wallet transactions.\n" + "\nHint: Use gettransaction for wallet transactions.\n" "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.\n", @@ -148,7 +152,11 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\" false \"myblockhash\"") + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); @@ -607,33 +615,54 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) return result; } +static std::string GetAllOutputTypes() +{ + std::string ret; + for (int i = TX_NONSTANDARD; i <= TX_WITNESS_UNKNOWN; ++i) { + if (i != TX_NONSTANDARD) ret += ", "; + ret += GetTxnOutputType(static_cast<txnouttype>(i)); + } + return ret; +} + static UniValue decodescript(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( - RPCHelpMan{"decodescript", + const RPCHelpMan help{"decodescript", "\nDecode a hex-encoded script.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, }, RPCResult{ "{\n" - " \"asm\":\"asm\", (string) Script public key\n" - " \"hex\":\"hex\", (string) hex-encoded public key\n" - " \"type\":\"type\", (string) The output type\n" - " \"reqSigs\": n, (numeric) The required signatures\n" - " \"addresses\": [ (json array of string)\n" - " \"address\" (string) bitcoin address\n" + " \"asm\":\"asm\", (string) Script public key\n" + " \"type\":\"type\", (string) The output type (e.g. "+GetAllOutputTypes()+")\n" + " \"reqSigs\": n, (numeric) The required signatures\n" + " \"addresses\": [ (json array of string)\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ],\n" - " \"p2sh\",\"address\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"p2sh\":\"str\" (string) address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH).\n" + " \"segwit\": { (json object) Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness).\n" + " \"asm\":\"str\", (string) String representation of the script public key\n" + " \"hex\":\"hexstr\", (string) Hex string of the script public key\n" + " \"type\":\"str\", (string) The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)\n" + " \"reqSigs\": n, (numeric) The required signatures (always 1)\n" + " \"addresses\": [ (json array of string) (always length 1)\n" + " \"address\" (string) segwit address\n" + " ,...\n" + " ],\n" + " \"p2sh-segwit\":\"str\" (string) address of the P2SH script wrapping this witness redeem script.\n" "}\n" }, RPCExamples{ HelpExampleCli("decodescript", "\"hexstring\"") + HelpExampleRpc("decodescript", "\"hexstring\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } RPCTypeCheck(request.params, {UniValue::VSTR}); @@ -645,7 +674,7 @@ static UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, false); + ScriptPubKeyToUniv(script, r, /* fIncludeHex */ false); UniValue type; type = find_value(r, "type"); @@ -679,7 +708,7 @@ static UniValue decodescript(const JSONRPCRequest& request) // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, true); + ScriptPubKeyToUniv(segwitScr, sr, /* fIncludeHex */ true); sr.pushKV("p2sh-segwit", EncodeDestination(CScriptID(segwitScr))); r.pushKV("segwit", sr); } @@ -789,23 +818,20 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) return EncodeHexTx(CTransaction(mergedTx)); } +// TODO(https://github.com/bitcoin/bitcoin/pull/10973#discussion_r267084237): +// This function is called from both wallet and node rpcs +// (signrawtransactionwithwallet and signrawtransactionwithkey). It should be +// moved to a util file so wallet code doesn't need to link against node code. +// Also the dependency on interfaces::Chain should be removed, so +// signrawtransactionwithkey doesn't need access to a Chain instance. UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, const UniValue& prevTxsUnival, CBasicKeyStore *keystore, bool is_temp_keystore, const UniValue& hashType) { // Fetch previous transactions (inputs): - CCoinsView viewDummy; - CCoinsViewCache view(&viewDummy); - { - LOCK2(cs_main, mempool.cs); - CCoinsViewCache &viewChain = *pcoinsTip; - CCoinsViewMemPool viewMempool(&viewChain, mempool); - view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view - - for (const CTxIn& txin : mtx.vin) { - view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. - } - - view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : mtx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. } + chain.findCoins(coins); // Add previous txouts given in the RPC call: if (!prevTxsUnival.isNull()) { @@ -837,10 +863,10 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con CScript scriptPubKey(pkData.begin(), pkData.end()); { - const Coin& coin = view.AccessCoin(out); - if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { + auto coin = coins.find(out); + if (coin != coins.end() && !coin->second.IsSpent() && coin->second.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin->second.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } @@ -851,7 +877,7 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } newcoin.nHeight = 1; - view.AddCoin(out, std::move(newcoin), true); + coins[out] = std::move(newcoin); } // if redeemScript and private keys were given, add redeemScript to the keystore so it can be signed @@ -895,15 +921,15 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con // Sign what we can: for (unsigned int i = 0; i < mtx.vin.size(); i++) { CTxIn& txin = mtx.vin[i]; - const Coin& coin = view.AccessCoin(txin.prevout); - if (coin.IsSpent()) { + auto coin = coins.find(txin.prevout); + if (coin == coins.end() || coin->second.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coin.out.scriptPubKey; - const CAmount& amount = coin.out.nValue; + const CScript& prevPubKey = coin->second.out.scriptPubKey; + const CAmount& amount = coin->second.out.nValue; - SignatureData sigdata = DataFromTransaction(mtx, i, coin.out); + SignatureData sigdata = DataFromTransaction(mtx, i, coin->second.out); // Only sign SIGHASH_SINGLE if there's a corresponding output: if (!fHashSingle || (i < mtx.vout.size())) { ProduceSignature(*keystore, MutableTransactionSignatureCreator(&mtx, i, amount, nHashType), prevPubKey, sigdata); @@ -913,7 +939,7 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con // amount must be specified for valid segwit signature if (amount == MAX_MONEY && !txin.scriptWitness.IsNull()) { - throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing amount for %s", coin.out.ToString())); + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing amount for %s", coin->second.out.ToString())); } ScriptError serror = SCRIPT_ERR_OK; @@ -964,7 +990,7 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -1021,24 +1047,14 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) return SignTransaction(*g_rpc_interfaces->chain, mtx, request.params[2], &keystore, true, request.params[3]); } -UniValue signrawtransaction(const JSONRPCRequest& request) -{ - // This method should be removed entirely in V0.19, along with the entries in the - // CRPCCommand table and rpc/client.cpp. - throw JSONRPCError(RPC_METHOD_DEPRECATED, "signrawtransaction was removed in v0.18.\n" - "Clients should transition to using signrawtransactionwithkey and signrawtransactionwithwallet"); -} - static UniValue sendrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( - RPCHelpMan{"sendrawtransaction", + const RPCHelpMan help{"sendrawtransaction", "\nSubmits raw transaction (serialized, hex-encoded) to local node and network.\n" "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, }, RPCResult{ "\"hex\" (string) The transaction hash in hex\n" @@ -1053,9 +1069,16 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendrawtransaction", "\"signedhex\"") }, - }.ToString()); + }; - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } + + RPCTypeCheck(request.params, { + UniValue::VSTR, + UniValueType(), // NUM or BOOL, checked later + }); // parse hex string from parameter CMutableTransaction mtx; @@ -1063,12 +1086,25 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - bool allowhighfees = false; - if (!request.params[1].isNull()) allowhighfees = request.params[1].get_bool(); + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; + // TODO: temporary migration code for old clients. Remove in v0.20 + if (request.params[1].isBool()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); + } else if (request.params[1].isNum()) { + size_t weight = GetTransactionWeight(*tx); + CFeeRate fr(AmountFromValue(request.params[1])); + // the +3/4 part rounds the value up, and is the same formula used when + // calculating the fee for a transaction + // (see GetVirtualTransactionSize) + max_raw_tx_fee = fr.GetFee((weight+3)/4); + } else if (!request.params[1].isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "second argument (maxfeerate) must be numeric"); + } + uint256 txid; - TransactionError err; std::string err_string; - if (!BroadcastTransaction(tx, txid, err, err_string, allowhighfees)) { + const TransactionError err = BroadcastTransaction(tx, txid, err_string, max_raw_tx_fee); + if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -1077,9 +1113,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) static UniValue testmempoolaccept(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"testmempoolaccept", + const RPCHelpMan help{"testmempoolaccept", "\nReturns result of mempool acceptance tests indicating if raw transaction (serialized, hex-encoded) would be accepted by mempool.\n" "\nThis checks if the transaction violates the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", @@ -1090,7 +1124,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, + {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE), "Reject transactions whose fee rate is higher than the specified value, expressed in " + CURRENCY_UNIT + "/kB\n"}, }, RPCResult{ "[ (array) The result of the mempool acceptance test for each raw transaction in the input array.\n" @@ -1112,10 +1146,17 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("testmempoolaccept", "[\"signedhex\"]") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); } - RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VBOOL}); + RPCTypeCheck(request.params, { + UniValue::VARR, + UniValueType(), // NUM or BOOL, checked later + }); + if (request.params[0].get_array().size() != 1) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Array must contain exactly one raw transaction for now"); } @@ -1127,9 +1168,19 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CAmount max_raw_tx_fee = ::maxTxFee; - if (!request.params[1].isNull() && request.params[1].get_bool()) { - max_raw_tx_fee = 0; + CAmount max_raw_tx_fee = DEFAULT_MAX_RAW_TX_FEE; + // TODO: temporary migration code for old clients. Remove in v0.20 + if (request.params[1].isBool()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); + } else if (request.params[1].isNum()) { + size_t weight = GetTransactionWeight(*tx); + CFeeRate fr(AmountFromValue(request.params[1])); + // the +3/4 part rounds the value up, and is the same formula used when + // calculating the fee for a transaction + // (see GetVirtualTransactionSize) + max_raw_tx_fee = fr.GetFee((weight+3)/4); + } else if (!request.params[1].isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "second argument (maxfeerate) must be numeric"); } UniValue result(UniValue::VARR); @@ -1493,12 +1544,11 @@ UniValue combinepsbt(const JSONRPCRequest& request) } PartiallySignedTransaction merged_psbt; - TransactionError error; - if (!CombinePSBTs(merged_psbt, error, psbtxs)) { + const TransactionError error = CombinePSBTs(merged_psbt, psbtxs); + if (error != TransactionError::OK) { throw JSONRPCTransactionError(error); } - UniValue result(UniValue::VOBJ); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); @@ -1860,21 +1910,21 @@ UniValue analyzepsbt(const JSONRPCRequest& request) " \"has_utxo\" : true|false (boolean) Whether a UTXO is provided\n" " \"is_final\" : true|false (boolean) Whether the input is finalized\n" " \"missing\" : { (json object, optional) Things that are missing that are required to complete this input\n" - " \"pubkeys\" : [ (array)\n" + " \"pubkeys\" : [ (array, optional)\n" " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose BIP 32 derivation path is missing\n" " ]\n" - " \"signatures\" : [ (array)\n" + " \"signatures\" : [ (array, optional)\n" " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose signature is missing\n" " ]\n" - " \"redeemscript\" : \"hash\" (string) Hash160 of the redeemScript that is missing\n" - " \"witnessscript\" : \"hash\" (string) SHA256 of the witnessScript that is missing\n" + " \"redeemscript\" : \"hash\" (string, optional) Hash160 of the redeemScript that is missing\n" + " \"witnessscript\" : \"hash\" (string, optional) SHA256 of the witnessScript that is missing\n" " }\n" - " \"next\" : \"role\" (string) Role of the next person that this input needs to go to\n" + " \"next\" : \"role\" (string, optional) Role of the next person that this input needs to go to\n" " }\n" " ,...\n" " ]\n" - " \"estimated_vsize\" : vsize (numeric) Estimated vsize of the final signed transaction\n" - " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"estimated_vsize\" : vsize (numeric, optional) Estimated vsize of the final signed transaction\n" + " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction in " + CURRENCY_UNIT + "/kB. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"fee\" : fee (numeric, optional) The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"next\" : \"role\" (string) Role of the next person that this psbt needs to go to\n" "}\n" @@ -1893,148 +1943,56 @@ UniValue analyzepsbt(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } - // Go through each input and build status + PSBTAnalysis psbta = AnalyzePSBT(psbtx); + UniValue result(UniValue::VOBJ); UniValue inputs_result(UniValue::VARR); - bool calc_fee = true; - bool all_final = true; - bool only_missing_sigs = true; - bool only_missing_final = false; - CAmount in_amt = 0; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; + for (const auto& input : psbta.inputs) { UniValue input_univ(UniValue::VOBJ); UniValue missing(UniValue::VOBJ); - // Check for a UTXO - CTxOut utxo; - if (psbtx.GetInputUTXO(utxo, i)) { - in_amt += utxo.nValue; - input_univ.pushKV("has_utxo", true); - } else { - input_univ.pushKV("has_utxo", false); - input_univ.pushKV("is_final", false); - input_univ.pushKV("next", "updater"); - calc_fee = false; - } + input_univ.pushKV("has_utxo", input.has_utxo); + input_univ.pushKV("is_final", input.is_final); + input_univ.pushKV("next", PSBTRoleName(input.next)); - // Check if it is final - if (!utxo.IsNull() && !PSBTInputSigned(input)) { - input_univ.pushKV("is_final", false); - all_final = false; - - // Figure out what is missing - SignatureData outdata; - bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); - - // Things are missing - if (!complete) { - if (!outdata.missing_pubkeys.empty()) { - // Missing pubkeys - UniValue missing_pubkeys_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_pubkeys) { - missing_pubkeys_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("pubkeys", missing_pubkeys_univ); - } - if (!outdata.missing_redeem_script.IsNull()) { - // Missing redeemScript - missing.pushKV("redeemscript", HexStr(outdata.missing_redeem_script)); - } - if (!outdata.missing_witness_script.IsNull()) { - // Missing witnessScript - missing.pushKV("witnessscript", HexStr(outdata.missing_witness_script)); - } - if (!outdata.missing_sigs.empty()) { - // Missing sigs - UniValue missing_sigs_univ(UniValue::VARR); - for (const CKeyID& pubkey : outdata.missing_sigs) { - missing_sigs_univ.push_back(HexStr(pubkey)); - } - missing.pushKV("signatures", missing_sigs_univ); - } - input_univ.pushKV("missing", missing); - - // If we are only missing signatures and nothing else, then next is signer - if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { - input_univ.pushKV("next", "signer"); - } else { - only_missing_sigs = false; - input_univ.pushKV("next", "updater"); - } - } else { - only_missing_final = true; - input_univ.pushKV("next", "finalizer"); + if (!input.missing_pubkeys.empty()) { + UniValue missing_pubkeys_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_pubkeys) { + missing_pubkeys_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("pubkeys", missing_pubkeys_univ); + } + if (!input.missing_redeem_script.IsNull()) { + missing.pushKV("redeemscript", HexStr(input.missing_redeem_script)); + } + if (!input.missing_witness_script.IsNull()) { + missing.pushKV("witnessscript", HexStr(input.missing_witness_script)); + } + if (!input.missing_sigs.empty()) { + UniValue missing_sigs_univ(UniValue::VARR); + for (const CKeyID& pubkey : input.missing_sigs) { + missing_sigs_univ.push_back(HexStr(pubkey)); } - } else if (!utxo.IsNull()){ - input_univ.pushKV("is_final", true); + missing.pushKV("signatures", missing_sigs_univ); + } + if (!missing.getKeys().empty()) { + input_univ.pushKV("missing", missing); } inputs_result.push_back(input_univ); } result.pushKV("inputs", inputs_result); - if (all_final) { - only_missing_sigs = false; - result.pushKV("next", "extractor"); + if (psbta.estimated_vsize != nullopt) { + result.pushKV("estimated_vsize", (int)*psbta.estimated_vsize); } - if (calc_fee) { - // Get the output amount - CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), 0, - [](int a, const CTxOut& b) { - return a += b.nValue; - } - ); - - // Get the fee - CAmount fee = in_amt - out_amt; - - // Estimate the size - CMutableTransaction mtx(*psbtx.tx); - CCoinsView view_dummy; - CCoinsViewCache view(&view_dummy); - bool success = true; - - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - PSBTInput& input = psbtx.inputs[i]; - if (SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true)) { - mtx.vin[i].scriptSig = input.final_script_sig; - mtx.vin[i].scriptWitness = input.final_script_witness; - - Coin newcoin; - if (!psbtx.GetInputUTXO(newcoin.out, i)) { - success = false; - break; - } - newcoin.nHeight = 1; - view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); - } else { - success = false; - break; - } - } - - if (success) { - CTransaction ctx = CTransaction(mtx); - size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); - result.pushKV("estimated_vsize", (int)size); - // Estimate fee rate - CFeeRate feerate(fee, size); - result.pushKV("estimated_feerate", feerate.ToString()); - } - result.pushKV("fee", ValueFromAmount(fee)); - - if (only_missing_sigs) { - result.pushKV("next", "signer"); - } else if (only_missing_final) { - result.pushKV("next", "finalizer"); - } else if (all_final) { - result.pushKV("next", "extractor"); - } else { - result.pushKV("next", "updater"); - } - } else { - result.pushKV("next", "updater"); + if (psbta.estimated_feerate != nullopt) { + result.pushKV("estimated_feerate", ValueFromAmount(psbta.estimated_feerate->GetFeePerK())); } + if (psbta.fee != nullopt) { + result.pushKV("fee", ValueFromAmount(*psbta.fee)); + } + result.pushKV("next", PSBTRoleName(psbta.next)); + return result; } @@ -2046,11 +2004,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, - { "hidden", "signrawtransaction", &signrawtransaction, {"hexstring","prevtxs","privkeys","sighashtype"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index cd90573da0..e803fabcc6 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -30,6 +30,7 @@ static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server starte static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers; +static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler); struct RPCCommandExecutionInfo { @@ -173,11 +174,11 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& { std::string strRet; std::string category; - std::set<rpcfn_type> setDone; + std::set<intptr_t> setDone; std::vector<std::pair<std::string, const CRPCCommand*> > vCommands; for (const auto& entry : mapCommands) - vCommands.push_back(make_pair(entry.second->category + entry.first, entry.second)); + vCommands.push_back(make_pair(entry.second.front()->category + entry.first, entry.second.front())); sort(vCommands.begin(), vCommands.end()); JSONRPCRequest jreq(helpreq); @@ -193,9 +194,9 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& jreq.strMethod = strMethod; try { - rpcfn_type pfn = pcmd->actor; - if (setDone.insert(pfn).second) - (*pfn)(jreq); + UniValue unused_result; + if (setDone.insert(pcmd->unique_id).second) + pcmd->actor(jreq, unused_result, true /* last_handler */); } catch (const std::exception& e) { @@ -337,32 +338,32 @@ CRPCTable::CRPCTable() const CRPCCommand *pcmd; pcmd = &vRPCCommands[vcidx]; - mapCommands[pcmd->name] = pcmd; + mapCommands[pcmd->name].push_back(pcmd); } } -const CRPCCommand *CRPCTable::operator[](const std::string &name) const -{ - std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); - if (it == mapCommands.end()) - return nullptr; - return (*it).second; -} - bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) { if (IsRPCRunning()) return false; - // don't allow overwriting for now - std::map<std::string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); - if (it != mapCommands.end()) - return false; - - mapCommands[name] = pcmd; + mapCommands[name].push_back(pcmd); return true; } +bool CRPCTable::removeCommand(const std::string& name, const CRPCCommand* pcmd) +{ + auto it = mapCommands.find(name); + if (it != mapCommands.end()) { + auto new_end = std::remove(it->second.begin(), it->second.end(), pcmd); + if (it->second.end() != new_end) { + it->second.erase(new_end, it->second.end()); + return true; + } + } + return false; +} + void StartRPC() { LogPrint(BCLog::RPC, "Starting RPC\n"); @@ -543,18 +544,28 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const } // Find method - const CRPCCommand *pcmd = tableRPC[request.strMethod]; - if (!pcmd) - throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found"); + auto it = mapCommands.find(request.strMethod); + if (it != mapCommands.end()) { + UniValue result; + for (const auto& command : it->second) { + if (ExecuteCommand(*command, request, result, &command == &it->second.back())) { + return result; + } + } + } + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found"); +} +static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler) +{ try { RPCCommandExecution execution(request.strMethod); // Execute, convert arguments to array if necessary if (request.params.isObject()) { - return pcmd->actor(transformNamedArguments(request, pcmd->argNames)); + return command.actor(transformNamedArguments(request, command.argNames), result, last_handler); } else { - return pcmd->actor(request); + return command.actor(request, result, last_handler); } } catch (const std::exception& e) diff --git a/src/rpc/server.h b/src/rpc/server.h index 2d62a76f3c..e2a85887ba 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -131,10 +131,31 @@ typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); class CRPCCommand { public: + //! RPC method handler reading request and assigning result. Should return + //! true if request is fully handled, false if it should be passed on to + //! subsequent handlers. + using Actor = std::function<bool(const JSONRPCRequest& request, UniValue& result, bool last_handler)>; + + //! Constructor taking Actor callback supporting multiple handlers. + CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::string> args, intptr_t unique_id) + : category(std::move(category)), name(std::move(name)), actor(std::move(actor)), argNames(std::move(args)), + unique_id(unique_id) + { + } + + //! Simplified constructor taking plain rpcfn_type function pointer. + CRPCCommand(const char* category, const char* name, rpcfn_type fn, std::initializer_list<const char*> args) + : CRPCCommand(category, name, + [fn](const JSONRPCRequest& request, UniValue& result, bool) { result = fn(request); return true; }, + {args.begin(), args.end()}, intptr_t(fn)) + { + } + std::string category; std::string name; - rpcfn_type actor; + Actor actor; std::vector<std::string> argNames; + intptr_t unique_id; }; /** @@ -143,10 +164,9 @@ public: class CRPCTable { private: - std::map<std::string, const CRPCCommand*> mapCommands; + std::map<std::string, std::vector<const CRPCCommand*>> mapCommands; public: CRPCTable(); - const CRPCCommand* operator[](const std::string& name) const; std::string help(const std::string& name, const JSONRPCRequest& helpreq) const; /** @@ -169,9 +189,7 @@ public: * * Returns false if RPC server is already running (dump concurrency protection). * - * Commands cannot be overwritten (returns false). - * - * Commands with different method names but the same callback function will + * Commands with different method names but the same unique_id will * be considered aliases, and only the first registered method name will * show up in the help text command listing. Aliased commands do not have * to have the same behavior. Server and client code can distinguish @@ -179,6 +197,7 @@ public: * register different names, types, and numbers of parameters. */ bool appendCommand(const std::string& name, const CRPCCommand* pcmd); + bool removeCommand(const std::string& name, const CRPCCommand* pcmd); }; bool IsDeprecatedRPCEnabled(const std::string& method); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 023b4b6746..10979b43b0 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,15 +1,12 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key_io.h> #include <keystore.h> -#include <policy/fees.h> -#include <rpc/protocol.h> #include <rpc/util.h> #include <tinyformat.h> #include <util/strencodings.h> -#include <validation.h> InitInterfaces* g_rpc_interfaces = nullptr; @@ -131,10 +128,9 @@ UniValue DescribeAddress(const CTxDestination& dest) return boost::apply_visitor(DescribeAddressVisitor(), dest); } -unsigned int ParseConfirmTarget(const UniValue& value) +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) { int target = value.get_int(); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); if (target < 1 || (unsigned int)target > max_target) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); } @@ -201,6 +197,7 @@ struct Sections { case RPCArg::Type::STR: case RPCArg::Type::NUM: case RPCArg::Type::AMOUNT: + case RPCArg::Type::RANGE: case RPCArg::Type::BOOL: { if (outer_type == OuterType::NAMED_ARG) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; @@ -315,6 +312,17 @@ std::string RPCExamples::ToDescriptionString() const return m_examples.empty() ? m_examples : "\nExamples:\n" + m_examples; } +bool RPCHelpMan::IsValidNumArgs(size_t num_args) const +{ + size_t num_required_args = 0; + for (size_t n = m_args.size(); n > 0; --n) { + if (!m_args.at(n - 1).IsOptional()) { + num_required_args = n; + break; + } + } + return num_required_args <= num_args && num_args <= m_args.size(); +} std::string RPCHelpMan::ToString() const { std::string ret; @@ -323,12 +331,7 @@ std::string RPCHelpMan::ToString() const ret += m_name; bool was_optional{false}; for (const auto& arg : m_args) { - bool optional; - if (arg.m_fallback.which() == 1) { - optional = true; - } else { - optional = RPCArg::Optional::NO != boost::get<RPCArg::Optional>(arg.m_fallback); - } + const bool optional = arg.IsOptional(); ret += " "; if (optional) { if (!was_optional) ret += "( "; @@ -370,6 +373,15 @@ std::string RPCHelpMan::ToString() const return ret; } +bool RPCArg::IsOptional() const +{ + if (m_fallback.which() == 1) { + return true; + } else { + return RPCArg::Optional::NO != boost::get<RPCArg::Optional>(m_fallback); + } +} + std::string RPCArg::ToDescriptionString() const { std::string ret; @@ -391,6 +403,10 @@ std::string RPCArg::ToDescriptionString() const ret += "numeric or string"; break; } + case Type::RANGE: { + ret += "numeric or array"; + break; + } case Type::BOOL: { ret += "boolean"; break; @@ -450,6 +466,8 @@ std::string RPCArg::ToStringObj(const bool oneline) const return res + "\"hex\""; case Type::NUM: return res + "n"; + case Type::RANGE: + return res + "n or [n,n]"; case Type::AMOUNT: return res + "amount"; case Type::BOOL: @@ -480,6 +498,7 @@ std::string RPCArg::ToString(const bool oneline) const return "\"" + m_name + "\""; } case Type::NUM: + case Type::RANGE: case Type::AMOUNT: case Type::BOOL: { return m_name; @@ -509,3 +528,17 @@ std::string RPCArg::ToString(const bool oneline) const } assert(false); } + +std::pair<int64_t, int64_t> ParseRange(const UniValue& value) +{ + if (value.isNum()) { + return {0, value.get_int64()}; + } + if (value.isArray() && value.size() == 2 && value[0].isNum() && value[1].isNum()) { + int64_t low = value[0].get_int64(); + int64_t high = value[1].get_int64(); + if (low > high) throw JSONRPCError(RPC_INVALID_PARAMETER, "Range specified as [begin,end] must not have begin after end"); + return {low, high}; + } + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified as end or as [begin,end]"); +} diff --git a/src/rpc/util.h b/src/rpc/util.h index 1c9ddcdf44..e4cc1fde44 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include <node/transaction.h> #include <pubkey.h> +#include <rpc/protocol.h> #include <script/standard.h> #include <univalue.h> @@ -32,11 +33,14 @@ CScript CreateMultisigRedeemscript(const int required, const std::vector<CPubKey UniValue DescribeAddress(const CTxDestination& dest); //! Parse a confirm target option and raise an RPC error if it is invalid. -unsigned int ParseConfirmTarget(const UniValue& value); +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); RPCErrorCode RPCErrorFromTransactionError(TransactionError terr); UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string = ""); +//! Parse a JSON range specified as int64, or [int64, int64] +std::pair<int64_t, int64_t> ParseRange(const UniValue& value); + struct RPCArg { enum class Type { OBJ, @@ -47,13 +51,14 @@ struct RPCArg { OBJ_USER_KEYS, //!< Special type where the user must set the keys e.g. to define multiple addresses; as opposed to e.g. an options object where the keys are predefined AMOUNT, //!< Special type representing a floating point amount (can be either NUM or STR) STR_HEX, //!< Special type that is a STR with only hex chars + RANGE, //!< Special type that is a NUM or [NUM,NUM] }; enum class Optional { /** Required arg */ NO, /** - * Optinal arg that is a named argument and has a default value of + * Optional arg that is a named argument and has a default value of * `null`. When possible, the default value should be specified. */ OMITTED_NAMED_ARG, @@ -110,6 +115,8 @@ struct RPCArg { assert(type == Type::ARR || type == Type::OBJ); } + bool IsOptional() const; + /** * Return the type string of the argument. * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description). @@ -185,6 +192,8 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples); std::string ToString() const; + /** If the supplied number of args is neither too small nor too high */ + bool IsValidNumArgs(size_t num_args) const; private: const std::string m_name; diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 625362f446..cd0c36d802 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -112,6 +112,12 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) BOOST_CHECK_EQUAL(block_filter.GetFilterType(), block_filter2.GetFilterType()); BOOST_CHECK_EQUAL(block_filter.GetBlockHash(), block_filter2.GetBlockHash()); BOOST_CHECK(block_filter.GetEncodedFilter() == block_filter2.GetEncodedFilter()); + + BlockFilter default_ctor_block_filter_1; + BlockFilter default_ctor_block_filter_2; + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetFilterType(), default_ctor_block_filter_2.GetFilterType()); + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetBlockHash(), default_ctor_block_filter_2.GetBlockHash()); + BOOST_CHECK(default_ctor_block_filter_1.GetEncodedFilter() == default_ctor_block_filter_2.GetEncodedFilter()); } BOOST_AUTO_TEST_CASE(blockfilters_json_test) diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 86cb00a78f..2cc38a0679 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -4,6 +4,7 @@ #include <crypto/aes.h> #include <crypto/chacha20.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -66,26 +67,6 @@ static void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, TestVector(CHMAC_SHA512(key.data(), key.size()), ParseHex(hexin), ParseHex(hexout)); } -static void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> buf, buf2; - - assert(key.size() == 16); - assert(in.size() == 16); - assert(correctout.size() == 16); - AES128Encrypt enc(key.data()); - buf.resize(correctout.size()); - buf2.resize(correctout.size()); - enc.Encrypt(buf.data(), in.data()); - BOOST_CHECK_EQUAL(HexStr(buf), HexStr(correctout)); - AES128Decrypt dec(key.data()); - dec.Decrypt(buf2.data(), buf.data()); - BOOST_CHECK_EQUAL(HexStr(buf2), HexStr(in)); -} - static void TestAES256(const std::string &hexkey, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -105,47 +86,6 @@ static void TestAES256(const std::string &hexkey, const std::string &hexin, cons BOOST_CHECK(buf == in); } -static void TestAES128CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) -{ - std::vector<unsigned char> key = ParseHex(hexkey); - std::vector<unsigned char> iv = ParseHex(hexiv); - std::vector<unsigned char> in = ParseHex(hexin); - std::vector<unsigned char> correctout = ParseHex(hexout); - std::vector<unsigned char> realout(in.size() + AES_BLOCKSIZE); - - // Encrypt the plaintext and verify that it equals the cipher - AES128CBCEncrypt enc(key.data(), iv.data(), pad); - int size = enc.Encrypt(in.data(), in.size(), realout.data()); - realout.resize(size); - BOOST_CHECK(realout.size() == correctout.size()); - BOOST_CHECK_MESSAGE(realout == correctout, HexStr(realout) + std::string(" != ") + hexout); - - // Decrypt the cipher and verify that it equals the plaintext - std::vector<unsigned char> decrypted(correctout.size()); - AES128CBCDecrypt dec(key.data(), iv.data(), pad); - size = dec.Decrypt(correctout.data(), correctout.size(), decrypted.data()); - decrypted.resize(size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(decrypted == in, HexStr(decrypted) + std::string(" != ") + hexin); - - // Encrypt and re-decrypt substrings of the plaintext and verify that they equal each-other - for(std::vector<unsigned char>::iterator i(in.begin()); i != in.end(); ++i) - { - std::vector<unsigned char> sub(i, in.end()); - std::vector<unsigned char> subout(sub.size() + AES_BLOCKSIZE); - int _size = enc.Encrypt(sub.data(), sub.size(), subout.data()); - if (_size != 0) - { - subout.resize(_size); - std::vector<unsigned char> subdecrypted(subout.size()); - _size = dec.Decrypt(subout.data(), subout.size(), subdecrypted.data()); - subdecrypted.resize(_size); - BOOST_CHECK(decrypted.size() == in.size()); - BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); - } - } -} - static void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) { std::vector<unsigned char> key = ParseHex(hexkey); @@ -200,6 +140,17 @@ static void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t see BOOST_CHECK(out == outres); } +static void TestPoly1305(const std::string &hexmessage, const std::string &hexkey, const std::string& hextag) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + std::vector<unsigned char> m = ParseHex(hexmessage); + std::vector<unsigned char> tag = ParseHex(hextag); + std::vector<unsigned char> tagres; + tagres.resize(POLY1305_TAGLEN); + poly1305_auth(tagres.data(), m.data(), m.size(), key.data()); + BOOST_CHECK(tag == tagres); +} + static std::string LongTestString() { std::string ret; for (int i=0; i<200000; i++) { @@ -428,14 +379,9 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { BOOST_AUTO_TEST_CASE(aes_testvectors) { // AES test vectors from FIPS 197. - TestAES128("000102030405060708090a0b0c0d0e0f", "00112233445566778899aabbccddeeff", "69c4e0d86a7b0430d8cdb78070b4c55a"); TestAES256("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "00112233445566778899aabbccddeeff", "8ea2b7ca516745bfeafc49904b496089"); // AES-ECB test vectors from NIST sp800-38a. - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "6bc1bee22e409f96e93d7e117393172a", "3ad77bb40d7a3660a89ecaf32466ef97"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "ae2d8a571e03ac9c9eb76fac45af8e51", "f5d3d58503b9699de785895a96fdbaaf"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "30c81c46a35ce411e5fbc1191a0a52ef", "43b1cd7f598ece23881b00e3ed030688"); - TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "f69f2445df4f9b17ad2b417be66c3710", "7b0c785e27e8ad3f8223207104725dd4"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "6bc1bee22e409f96e93d7e117393172a", "f3eed1bdb5d2a03c064b5a7e3db181f8"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "ae2d8a571e03ac9c9eb76fac45af8e51", "591ccb10d410ed26dc5ba74a31362870"); TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "30c81c46a35ce411e5fbc1191a0a52ef", "b6ed21b99ca6f4f9f153e7b1beafed1d"); @@ -443,27 +389,6 @@ BOOST_AUTO_TEST_CASE(aes_testvectors) { } BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { - - // NIST AES CBC 128-bit encryption test-vectors - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", false, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", false, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", false, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", false, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a7"); - - // The same vectors with padding enabled - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", true, \ - "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d8964e0b149c10b7b682e6e39aaeb731c"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", true, \ - "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b255e21d7100b988ffec32feeafaf23538"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", true, \ - "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516f6eccda327bf8e5ec43718b0039adceb"); - TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", true, \ - "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a78cb82807230e1321d3fae00d18cc2012"); - // NIST AES CBC 256-bit encryption test-vectors TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ "000102030405060708090A0B0C0D0E0F", false, "6bc1bee22e409f96e93d7e117393172a", \ @@ -524,6 +449,76 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) "fab78c9"); } +BOOST_AUTO_TEST_CASE(poly1305_testvector) +{ + // RFC 7539, section 2.5.2. + TestPoly1305("43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"); + + // RFC 7539, section A.3. + TestPoly1305("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"); + + TestPoly1305("416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"); + + TestPoly1305("2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"); + + TestPoly1305("ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"); + + TestPoly1305("02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"); + + TestPoly1305("fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"); + + TestPoly1305("fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"); + + TestPoly1305("e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"); +} + BOOST_AUTO_TEST_CASE(countbits_tests) { FastRandomContext ctx; diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 index 0000000000..079a09f8f9 --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <flatfile.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) +{ + auto data_dir = SetDataDir("flatfile_test"); + + FlatFilePos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1("A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but the main benefits are " + "lost if a trusted third party is still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(FlatFilePos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(FlatFilePos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(FlatFilePos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index ad62a5faf0..8b03a7e46e 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -10,8 +10,6 @@ #include <vector> -const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; - void test_one_input(std::vector<uint8_t> buffer); #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/main.cpp b/src/test/main.cpp new file mode 100644 index 0000000000..ff3f36b561 --- /dev/null +++ b/src/test/main.cpp @@ -0,0 +1,7 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#define BOOST_TEST_MODULE Bitcoin Core Test Suite + +#include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index ff48398925..9bb2bf551b 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -31,10 +31,9 @@ UniValue CallRPC(std::string args) request.strMethod = strMethod; request.params = RPCConvertValues(strMethod, vArgs); request.fHelp = false; - BOOST_CHECK(tableRPC[strMethod]); - rpcfn_type method = tableRPC[strMethod]->actor; + if (RPCIsInWarmup(nullptr)) SetRPCWarmupFinished(); try { - UniValue result = (*method)(request); + UniValue result = tableRPC.execute(request); return result; } catch (const UniValue& objError) { diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 0c3fb7c398..cdfd4db589 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -66,36 +66,36 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha { SetDataDir("tempdir"); const CChainParams& chainparams = Params(); - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - - RegisterAllCoreRPCCommands(tableRPC); - ClearDatadirCache(); - - // We have to run a scheduler thread to prevent ActivateBestChain - // from blocking due to queue overrun. - threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); - GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - - mempool.setSanityCheck(1.0); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); - pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - if (!LoadGenesisBlock(chainparams)) { - throw std::runtime_error("LoadGenesisBlock failed."); - } - { - CValidationState state; - if (!ActivateBestChain(state, chainparams)) { - throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); - } - } - nScriptCheckThreads = 3; - for (int i=0; i < nScriptCheckThreads-1; i++) - threadGroup.create_thread(&ThreadScriptCheck); - - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + + RegisterAllCoreRPCCommands(tableRPC); + ClearDatadirCache(); + + // We have to run a scheduler thread to prevent ActivateBestChain + // from blocking due to queue overrun. + threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + + mempool.setSanityCheck(1.0); + pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); + if (!LoadGenesisBlock(chainparams)) { + throw std::runtime_error("LoadGenesisBlock failed."); + } + + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); + } + + nScriptCheckThreads = 3; + for (int i = 0; i < nScriptCheckThreads - 1; i++) + threadGroup.create_thread(&ThreadScriptCheck); + + g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. } TestingSetup::~TestingSetup() diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 4a06845683..38c6d85a8d 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -71,10 +71,6 @@ private: /** Testing setup that configures a complete environment. * Included are data directory, coins database, script check threads setup. */ -class CConnman; -class CNode; - -class PeerLogicValidation; struct TestingSetup : public BasicTestingSetup { boost::thread_group threadGroup; CScheduler scheduler; diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp deleted file mode 100644 index 46b63b93b4..0000000000 --- a/src/test/test_bitcoin_main.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#define BOOST_TEST_MODULE Bitcoin Test Suite - -#include <banman.h> -#include <net.h> - -#include <memory> - -#include <boost/test/unit_test.hpp> - -std::unique_ptr<CConnman> g_connman; -std::unique_ptr<BanMan> g_banman; - -[[noreturn]] void Shutdown(void* parg) -{ - std::exit(EXIT_SUCCESS); -} - -[[noreturn]] void StartShutdown() -{ - std::exit(EXIT_SUCCESS); -} - -bool ShutdownRequested() -{ - return false; -} diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 860f64bb11..e106bdeb65 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -78,80 +78,40 @@ BOOST_AUTO_TEST_CASE(util_HexStr) "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected + 5, true), - "04 67 8a fd b0"); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_expected + sizeof(ParseHex_expected), ParseHex_expected + sizeof(ParseHex_expected)), ""); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected + sizeof(ParseHex_expected), - ParseHex_expected + sizeof(ParseHex_expected), true), - ""); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_expected, ParseHex_expected), ""); - BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected, true), - ""); - std::vector<unsigned char> ParseHex_vec(ParseHex_expected, ParseHex_expected + 5); BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec, true), - "04 67 8a fd b0"); - - BOOST_CHECK_EQUAL( HexStr(ParseHex_vec.rbegin(), ParseHex_vec.rend()), "b0fd8a6704" ); BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec.rbegin(), ParseHex_vec.rend(), true), - "b0 fd 8a 67 04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 1), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "04" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 1), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 5), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "b0fd8a6704" ); BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 5), - std::reverse_iterator<const uint8_t *>(ParseHex_expected), true), - "b0 fd 8a 67 04" - ); - - BOOST_CHECK_EQUAL( HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 65), std::reverse_iterator<const uint8_t *>(ParseHex_expected)), "5f1df16b2b704c8a578d0bbaf74d385cde12c11ee50455f3c438ef4c3fbcf649b6de611feae06279a60939e028a8d65c10b73071a6f16719274855feb0fd8a6704" @@ -180,9 +140,10 @@ struct TestArgsManager : public ArgsManager { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } std::string error; - BOOST_REQUIRE(ReadConfigStream(streamConfig, error)); + BOOST_REQUIRE(ReadConfigStream(streamConfig, "", error)); } void SetNetworkOnlyArg(const std::string arg) { diff --git a/src/test/main_tests.cpp b/src/test/validation_tests.cpp index 5b3f2bc578..8d06ecd3a9 100644 --- a/src/test/main_tests.cpp +++ b/src/test/validation_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <validation.h> #include <net.h> +#include <validation.h> #include <test/test_bitcoin.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(main_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_tests, TestingSetup) static void TestBlockSubsidyHalvings(const Consensus::Params& consensusParams) { diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 68f47d5cce..ca556bdc7b 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -764,7 +764,7 @@ std::vector<CTxMemPool::indexed_transaction_set::const_iterator> CTxMemPool::Get return iters; } -void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) +void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) const { LOCK(cs); auto iters = GetSortedDepthAndScore(); diff --git a/src/txmempool.h b/src/txmempool.h index f7afaec8fc..a8a0f7fa45 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -184,7 +184,7 @@ private: const LockPoints& lp; }; -// extracts a transaction hash from CTxMempoolEntry or CTransactionRef +// extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -588,7 +588,7 @@ public: void clear(); void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); - void queryHashes(std::vector<uint256>& vtxid); + void queryHashes(std::vector<uint256>& vtxid) const; bool isSpent(const COutPoint& outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 947d7e2308..16ab24686b 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -52,7 +52,7 @@ void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_s void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } -void CClientUIInterface::LoadWallet(std::shared_ptr<CWallet> wallet) { return g_ui_signals.LoadWallet(wallet); } +void CClientUIInterface::LoadWallet(std::unique_ptr<interfaces::Wallet>& wallet) { return g_ui_signals.LoadWallet(wallet); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } diff --git a/src/ui_interface.h b/src/ui_interface.h index fe466b3ca4..f1aebce3bb 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -11,7 +11,6 @@ #include <stdint.h> #include <string> -class CWallet; class CBlockIndex; namespace boost { namespace signals2 { @@ -19,6 +18,10 @@ class connection; } } // namespace boost +namespace interfaces { +class Wallet; +} // namespace interfaces + /** General change type (added, updated, removed). */ enum ChangeType { @@ -102,7 +105,7 @@ public: ADD_SIGNALS_DECL_WRAPPER(NotifyAlertChanged, void, ); /** A wallet has been loaded. */ - ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::shared_ptr<CWallet> wallet); + ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::unique_ptr<interfaces::Wallet>& wallet); /** * Show progress e.g. for verifychain. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index cf77044094..7c4364a082 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -12,6 +12,7 @@ #include <attributes.h> #include <cstdint> +#include <iterator> #include <string> #include <vector> @@ -121,28 +122,25 @@ NODISCARD bool ParseUInt64(const std::string& str, uint64_t *out); NODISCARD bool ParseDouble(const std::string& str, double *out); template<typename T> -std::string HexStr(const T itbegin, const T itend, bool fSpaces=false) +std::string HexStr(const T itbegin, const T itend) { std::string rv; static const char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - rv.reserve((itend-itbegin)*3); + rv.reserve(std::distance(itbegin, itend) * 2); for(T it = itbegin; it < itend; ++it) { unsigned char val = (unsigned char)(*it); - if(fSpaces && it != itbegin) - rv.push_back(' '); rv.push_back(hexmap[val>>4]); rv.push_back(hexmap[val&15]); } - return rv; } template<typename T> -inline std::string HexStr(const T& vch, bool fSpaces=false) +inline std::string HexStr(const T& vch) { - return HexStr(vch.begin(), vch.end(), fSpaces); + return HexStr(vch.begin(), vch.end()); } /** diff --git a/src/util/system.cpp b/src/util/system.cpp index 27ed24d012..9594dd81bf 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -74,7 +74,6 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; -const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; ArgsManager gArgs; @@ -136,6 +135,14 @@ bool DirIsWritable(const fs::path& directory) return true; } +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) +{ + constexpr uint64_t min_disk_space = 52428800; // 50 MiB + + uint64_t free_bytes_available = fs::space(dir).available; + return free_bytes_available >= min_disk_space + additional_bytes; +} + /** * Interpret a string argument as a boolean. * @@ -354,8 +361,7 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const return unsuitables; } - -const std::set<std::string> ArgsManager::GetUnrecognizedSections() const +const std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ @@ -363,14 +369,11 @@ const std::set<std::string> ArgsManager::GetUnrecognizedSections() const CBaseChainParams::TESTNET, CBaseChainParams::MAIN }; - std::set<std::string> diff; LOCK(cs_args); - std::set_difference( - m_config_sections.begin(), m_config_sections.end(), - available_sections.begin(), available_sections.end(), - std::inserter(diff, diff.end())); - return diff; + std::list<SectionInfo> unrecognized = m_config_sections; + unrecognized.remove_if([](const SectionInfo& appeared){ return available_sections.find(appeared.m_name) != available_sections.end(); }); + return unrecognized; } void ArgsManager::SelectConfigNetwork(const std::string& network) @@ -794,7 +797,7 @@ static std::string TrimString(const std::string& str, const std::string& pattern return str.substr(front, end - front + 1); } -static bool GetConfigOptions(std::istream& stream, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::set<std::string>& sections) +static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; std::string::size_type pos; @@ -810,7 +813,7 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect if (!str.empty()) { if (*str.begin() == '[' && *str.rbegin() == ']') { const std::string section = str.substr(1, str.size() - 2); - sections.insert(section); + sections.emplace_back(SectionInfo{section, filepath, linenr}); prefix = section + '.'; } else if (*str.begin() == '-') { error = strprintf("parse error on line %i: %s, options in configuration file must be specified without leading -", linenr, str); @@ -823,8 +826,8 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return false; } options.emplace_back(name, value); - if ((pos = name.rfind('.')) != std::string::npos) { - sections.insert(name.substr(0, pos)); + if ((pos = name.rfind('.')) != std::string::npos && prefix.length() <= pos) { + sections.emplace_back(SectionInfo{name.substr(0, pos), filepath, linenr}); } } else { error = strprintf("parse error on line %i: %s", linenr, str); @@ -839,12 +842,11 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return true; } -bool ArgsManager::ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys) +bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); std::vector<std::pair<std::string, std::string>> options; - m_config_sections.clear(); - if (!GetConfigOptions(stream, error, options, m_config_sections)) { + if (!GetConfigOptions(stream, filepath, error, options, m_config_sections)) { return false; } for (const std::pair<std::string, std::string>& option : options) { @@ -875,6 +877,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); @@ -882,7 +885,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) // ok to not have a config file if (stream.good()) { - if (!ReadConfigStream(stream, error, ignore_invalid_keys)) { + if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } // if there is an -includeconf in the override args, but it is empty, that means the user @@ -913,7 +916,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) for (const std::string& to_include : includeconf) { fsbridge::ifstream include_config(GetConfigFile(to_include)); if (include_config.good()) { - if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) { + if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { return false; } LogPrintf("Included configuration file %s\n", to_include.c_str()); @@ -965,23 +968,6 @@ std::string ArgsManager::GetChainName() const return CBaseChainParams::MAIN; } -#ifndef WIN32 -fs::path GetPidFile() -{ - return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); -} - -void CreatePidFile(const fs::path &path, pid_t pid) -{ - FILE* file = fsbridge::fopen(path, "w"); - if (file) - { - fprintf(file, "%d\n", pid); - fclose(file); - } -} -#endif - bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 diff --git a/src/util/system.h b/src/util/system.h index 8867e49478..54eb88e261 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -40,7 +40,6 @@ int64_t GetStartupTime(); extern const char * const BITCOIN_CONF_FILENAME; -extern const char * const BITCOIN_PID_FILENAME; /** Translate a message to the native language of the user. */ const extern std::function<std::string(const char*)> G_TRANSLATION_FUN; @@ -73,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. @@ -86,10 +86,6 @@ const fs::path &GetBlocksDir(); const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); fs::path GetConfigFile(const std::string& confPath); -#ifndef WIN32 -fs::path GetPidFile(); -void CreatePidFile(const fs::path &path, pid_t pid); -#endif #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif @@ -132,6 +128,13 @@ enum class OptionsCategory { HIDDEN // Always the last option to avoid printing these in the help }; +struct SectionInfo +{ + std::string m_name; + std::string m_file; + int m_line; +}; + class ArgsManager { protected: @@ -152,9 +155,9 @@ protected: std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); - std::set<std::string> m_config_sections GUARDED_BY(cs_args); + std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); - NODISCARD bool ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys = false); + NODISCARD bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); public: ArgsManager(); @@ -178,7 +181,7 @@ public: /** * Log warnings for unrecognized section names in the config file. */ - const std::set<std::string> GetUnrecognizedSections() const; + const std::list<SectionInfo> GetUnrecognizedSections() const; /** * Return a vector of strings of the given argument diff --git a/src/validation.cpp b/src/validation.cpp index 1806bc1268..ae3985485e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -15,6 +15,7 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <cuckoocache.h> +#include <flatfile.h> #include <hash.h> #include <index/txindex.h> #include <policy/fees.h> @@ -35,9 +36,9 @@ #include <txmempool.h> #include <ui_interface.h> #include <undo.h> -#include <util/system.h> #include <util/moneystr.h> #include <util/strencodings.h> +#include <util/system.h> #include <validationinterface.h> #include <warnings.h> @@ -165,7 +166,7 @@ public: * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. */ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); @@ -177,7 +178,7 @@ public: // Manual block validity manipulation: bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); - bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool ReplayBlocks(const CChainParams& params, CCoinsView* view); @@ -204,10 +205,12 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Mark a block as not having block data + void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); } g_chainstate; /** @@ -317,7 +320,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState & static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -1042,7 +1047,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus // CBlock and CBlockIndex // -static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1063,7 +1068,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1089,7 +1094,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - CDiskBlockPos blockPos; + FlatFilePos blockPos; { LOCK(cs_main); blockPos = pindex->GetBlockPos(); @@ -1103,9 +1108,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos hpos = pos; + FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { @@ -1140,7 +1145,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos block_pos; + FlatFilePos block_pos; { LOCK(cs_main); block_pos = pindex->GetBlockPos(); @@ -1450,9 +1455,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi return true; } -namespace { - -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1479,9 +1482,9 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return true; } -static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) { - CDiskBlockPos pos = pindex->GetUndoPos(); + FlatFilePos pos = pindex->GetUndoPos(); if (pos.IsNull()) { return error("%s: no undo data available", __func__); } @@ -1528,8 +1531,6 @@ static bool AbortNode(CValidationState& state, const std::string& strMessage, co return state.Error(strMessage); } -} // namespace - /** * Restore the UTXO in a Coin at a given COutPoint * @param undo The Coin to be restored. @@ -1627,37 +1628,24 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; + FlatFilePos _pos; if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) @@ -2134,8 +2122,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0, true)) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). @@ -2168,8 +2157,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); @@ -2259,12 +2249,6 @@ void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainPar } if (nUpgraded > 0) AppendWarning(warningMessages, strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); - if (nUpgraded > 100/2) - { - std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); - // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - DoWarning(strWarning); - } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, /* Continued */ pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, @@ -2640,6 +2624,14 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { } } +static void LimitValidationInterfaceQueue() { + AssertLockNotHeld(cs_main); + + if (GetMainSignals().CallbacksPending() > 10) { + SyncWithValidationInterfaceQueue(); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either nullptr or a pointer to a block @@ -2668,15 +2660,13 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& do { boost::this_thread::interruption_point(); - if (GetMainSignals().CallbacksPending() > 10) { - // Block until the validation queue drains. This should largely - // never happen in normal operation, however may happen during - // reindex, causing memory blowup if we run too far ahead. - // Note that if a validationinterface callback ends up calling - // ActivateBestChain this may lead to a deadlock! We should - // probably have a DEBUG_LOCKORDER test for this in the future. - SyncWithValidationInterfaceQueue(); - } + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. + LimitValidationInterfaceQueue(); { LOCK(cs_main); @@ -2787,64 +2777,85 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { - AssertLockHeld(cs_main); + CBlockIndex* to_mark_failed = pindex; + bool pindex_was_in_chain = false; + int disconnected = 0; - // We first disconnect backwards and then mark the blocks as invalid. - // This prevents a case where pruned nodes may fail to invalidateblock - // and be left unable to start as they have no tip candidates (as there - // are no blocks that meet the "have data and are not invalid per - // nStatus" criteria for inclusion in setBlockIndexCandidates). + // Disconnect (descendants of) pindex, and mark them invalid. + while (true) { + if (ShutdownRequested()) break; - bool pindex_was_in_chain = false; - CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); - DisconnectedBlockTransactions disconnectpool; - while (chainActive.Contains(pindex)) { + LOCK(cs_main); + if (!chainActive.Contains(pindex)) break; pindex_was_in_chain = true; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams, &disconnectpool)) { - // It's probably hopeless to try to make the mempool consistent - // here if DisconnectTip failed, but we can try. - UpdateMempoolForReorg(disconnectpool, false); - return false; - } - } - - // Now mark the blocks we just disconnected as descendants invalid - // (note this may not be all descendants). - while (pindex_was_in_chain && invalid_walk_tip != pindex) { - invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + DisconnectedBlockTransactions disconnectpool; + bool ret = DisconnectTip(state, chainparams, &disconnectpool); + // DisconnectTip will add transactions to disconnectpool. + // Adjust the mempool to be consistent with the new tip, adding + // transactions back to the mempool if disconnecting was succesful, + // and we're not doing a very deep invalidation (in which case + // keeping the mempool up to date is probably futile anyway). + UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + if (!ret) return false; + assert(invalid_walk_tip->pprev == chainActive.Tip()); + + // We immediately mark the disconnected blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(invalid_walk_tip); setBlockIndexCandidates.erase(invalid_walk_tip); - invalid_walk_tip = invalid_walk_tip->pprev; + setBlockIndexCandidates.insert(invalid_walk_tip->pprev); + if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { + // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children + // need to be BLOCK_FAILED_CHILD instead. + to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(to_mark_failed); + } + + // Track the last disconnected block, so we can correct its BLOCK_FAILED_CHILD status in future + // iterations, or, if it's the last one, call InvalidChainFound on it. + to_mark_failed = invalid_walk_tip; } - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); - m_failed_blocks.insert(pindex); + { + LOCK(cs_main); + if (chainActive.Contains(to_mark_failed)) { + // If the to-be-marked invalid block is in the active chain, something is interfering and we can't proceed. + return false; + } - // DisconnectTip will add transactions to disconnectpool; try to add these - // back to the mempool. - UpdateMempoolForReorg(disconnectpool, true); + // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain + to_mark_failed->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(to_mark_failed); + setBlockIndexCandidates.erase(to_mark_failed); + m_failed_blocks.insert(to_mark_failed); - // The resulting new best tip may not be in setBlockIndexCandidates anymore, so - // add it again. - BlockMap::iterator it = mapBlockIndex.begin(); - while (it != mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { - setBlockIndexCandidates.insert(it->second); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so + // add it again. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; } - it++; - } - InvalidChainFound(pindex); + InvalidChainFound(to_mark_failed); + } // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); + uiInterface.NotifyBlockTip(IsInitialBlockDownload(), to_mark_failed->pprev); } return true; } @@ -2928,7 +2939,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2974,7 +2985,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3009,21 +3020,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3031,32 +3034,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int return true; } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; @@ -3441,26 +3435,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - CDiskBlockPos blockPos; + FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { error("%s: FindBlockPos failed", __func__); - return CDiskBlockPos(); + return FlatFilePos(); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); - return CDiskBlockPos(); + return FlatFilePos(); } } return blockPos; } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3522,7 +3516,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali // Write block to history file if (fNewBlock) *fNewBlock = true; try { - CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3653,9 +3647,9 @@ void PruneOneBlockFile(const int fileNumber) void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3763,52 +3757,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) +static FlatFileSeq BlockFileSeq() { - uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); } -static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +static FlatFileSeq UndoFileSeq() { - if (pos.IsNull()) - return nullptr; - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) - file = fsbridge::fopen(path, "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } - } - return file; + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { + return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) @@ -3915,7 +3885,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -4168,38 +4138,114 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) { return g_chainstate.ReplayBlocks(params, view); } -bool CChainState::RewindBlockIndex(const CChainParams& params) +//! Helper for CChainState::RewindBlockIndex +void CChainState::EraseBlockData(CBlockIndex* index) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(!chainActive.Contains(index)); // Make sure this block isn't active + + // Reduce validity + index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + index->nFile = 0; + index->nDataPos = 0; + index->nUndoPos = 0; + // Remove various other things + index->nTx = 0; + index->nChainTx = 0; + index->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(index); + // Update indexes + setBlockIndexCandidates.erase(index); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(index->pprev); + while (ret.first != ret.second) { + if (ret.first->second == index) { + mapBlocksUnlinked.erase(ret.first++); + } else { + ++ret.first; + } + } + // Mark parent as eligible for main chain again + if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { + setBlockIndexCandidates.insert(index->pprev); + } +} +bool CChainState::RewindBlockIndex(const CChainParams& params) +{ // Note that during -reindex-chainstate we are called with an empty chainActive! - int nHeight = 1; - while (nHeight <= chainActive.Height()) { - // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all - // blocks in ConnectBlock, we don't need to go back and - // re-download/re-verify blocks from before segwit actually activated. - if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; + // First erase all post-segwit blocks without witness not in the main chain, + // as this can we done without costly DisconnectTip calls. Active + // blocks will be dealt with below (releasing cs_main in between). + { + LOCK(cs_main); + for (const auto& entry : mapBlockIndex) { + if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(entry.second)) { + EraseBlockData(entry.second); + } } - nHeight++; } + // Find what height we need to reorganize to. + CBlockIndex *tip; + int nHeight = 1; + { + LOCK(cs_main); + while (nHeight <= chainActive.Height()) { + // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all + // blocks in ConnectBlock, we don't need to go back and + // re-download/re-verify blocks from before segwit actually activated. + if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } + + tip = chainActive.Tip(); + } // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; - CBlockIndex* pindex = chainActive.Tip(); - while (chainActive.Height() >= nHeight) { - if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; - } - if (!DisconnectTip(state, params, nullptr)) { - return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", pindex->nHeight, FormatStateMessage(state)); + // Loop until the tip is below nHeight, or we reach a pruned block. + while (!ShutdownRequested()) { + { + LOCK(cs_main); + // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) + assert(tip == chainActive.Tip()); + if (tip == nullptr || tip->nHeight < nHeight) break; + if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + + // Disconnect block + if (!DisconnectTip(state, params, nullptr)) { + return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, FormatStateMessage(state)); + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + EraseBlockData(tip); + + tip = tip->pprev; } + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); + // Occasionally flush state to disk. if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); @@ -4207,53 +4253,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) } } - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (const auto& entry : mapBlockIndex) { - CBlockIndex* pindexIter = entry.second; - - // Note: If we encounter an insufficiently validated block that - // is on chainActive, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on chainActive at this point - // must not have their validity reduced. - if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { - // Reduce validity - pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - pindexIter->nFile = 0; - pindexIter->nDataPos = 0; - pindexIter->nUndoPos = 0; - // Remove various other things - pindexIter->nTx = 0; - pindexIter->nChainTx = 0; - pindexIter->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(pindexIter); - // Update indexes - setBlockIndexCandidates.erase(pindexIter); - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); - while (ret.first != ret.second) { - if (ret.first->second == pindexIter) { - mapBlocksUnlinked.erase(ret.first++); - } else { - ++ret.first; - } - } - } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->HaveTxsDownloaded()) { - setBlockIndexCandidates.insert(pindexIter); - } - } - - if (chainActive.Tip() != nullptr) { - // We can't prune block index candidates based on our tip if we have - // no tip due to chainActive being empty! - PruneBlockIndexCandidates(); + { + LOCK(cs_main); + if (chainActive.Tip() != nullptr) { + // We can't prune block index candidates based on our tip if we have + // no tip due to chainActive being empty! + PruneBlockIndexCandidates(); - CheckBlockIndex(params.GetConsensus()); + CheckBlockIndex(params.GetConsensus()); + } } return true; @@ -4348,7 +4356,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) try { const CBlock& block = chainparams.GenesisBlock(); - CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); @@ -4365,10 +4373,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return g_chainstate.LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; + static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4454,9 +4462,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { diff --git a/src/validation.h b/src/validation.h index 1975846b69..d84e3a0dbc 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include <versionbits.h> #include <algorithm> +#include <atomic> #include <exception> #include <map> #include <memory> @@ -30,10 +31,9 @@ #include <utility> #include <vector> -#include <atomic> - class CBlockIndex; class CBlockTreeDB; +class CBlockUndo; class CChainParams; class CCoinsViewDB; class CInv; @@ -181,9 +181,6 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Minimum disk space required - used in CheckDiskSpace() */ -static const uint64_t nMinDiskSpace = 52428800; - /** Pruning-related variables and constants */ /** True if any block files have ever been pruned. */ extern bool fHavePruned; @@ -245,14 +242,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); -/** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, @@ -391,11 +386,13 @@ void InitScriptExecutionCache(); /** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ @@ -448,7 +445,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 99d880daa0..6a326bfd97 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -84,6 +84,14 @@ bool IsWalletLoaded(const fs::path& wallet_path) return database && database->IsDatabaseLoaded(database_filename); } +fs::path WalletDataFilePath(const fs::path& wallet_path) +{ + fs::path env_directory; + std::string database_filename; + SplitWalletPath(wallet_path, env_directory, database_filename); + return env_directory / database_filename; +} + /** * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. diff --git a/src/wallet/db.h b/src/wallet/db.h index 9df965305a..762fb83a2f 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -101,6 +101,9 @@ public: /** Return whether a wallet database is currently loaded. */ bool IsWalletLoaded(const fs::path& wallet_path); +/** Given a wallet directory path or legacy file path, return path to main data file in the wallet database. */ +fs::path WalletDataFilePath(const fs::path& wallet_path); + /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 7a71aea715..ef7f3be728 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -27,9 +27,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } { - LOCK(mempool.cs); - auto it_mp = mempool.mapTx.find(wtx.GetHash()); - if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + if (wallet->chain().hasDescendantsInMempool(wtx.GetHash())) { errors.push_back("Transaction has descendants in the mempool"); return feebumper::Result::INVALID_PARAMETER; } @@ -125,16 +123,17 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay // fee that our node may not be aware of. + CFeeRate nodeIncrementalRelayFee = wallet->chain().relayIncrementalFee(); CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); - if (::incrementalRelayFee > walletIncrementalRelayFee) { - walletIncrementalRelayFee = ::incrementalRelayFee; + if (nodeIncrementalRelayFee > walletIncrementalRelayFee) { + walletIncrementalRelayFee = nodeIncrementalRelayFee; } if (total_fee > 0) { - CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); + CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + nodeIncrementalRelayFee.GetFee(maxNewTxSize); if (total_fee < minTotalFee) { errors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", - FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); + FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(nodeIncrementalRelayFee.GetFee(maxNewTxSize)))); return Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(*wallet, maxNewTxSize); @@ -146,7 +145,7 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin new_fee = total_fee; nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); } else { - new_fee = GetMinimumFee(*wallet, maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); + new_fee = GetMinimumFee(*wallet, maxNewTxSize, coin_control, nullptr /* FeeCalculation */); nNewFeeRate = CFeeRate(new_fee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate @@ -161,9 +160,10 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin } // Check that in all cases the new fee doesn't violate maxTxFee - if (new_fee > maxTxFee) { + const CAmount max_tx_fee = wallet->chain().maxTxFee(); + if (new_fee > max_tx_fee) { errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(new_fee), FormatMoney(maxTxFee))); + FormatMoney(new_fee), FormatMoney(max_tx_fee))); return Result::WALLET_ERROR; } @@ -197,7 +197,7 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; - if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet, ::feeEstimator))) { + if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet))) { wallet->WalletLogPrintf("Bumping fee and discarding dust output\n"); new_fee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); @@ -247,7 +247,7 @@ Result CommitTransaction(CWallet* wallet, const uint256& txid, CMutableTransacti CReserveKey reservekey(wallet); CValidationState state; - if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, g_connman.get(), state)) { + if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, state)) { // NOTE: CommitTransaction never returns false, so this should never happen. errors.push_back(strprintf("The transaction was rejected: %s", FormatStateMessage(state))); return Result::WALLET_ERROR; diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 9e2984ff05..560c86a70a 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -6,7 +6,6 @@ #include <wallet/fees.h> #include <policy/policy.h> -#include <txmempool.h> #include <util/system.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -19,12 +18,13 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes) } -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc) { - CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, pool, estimator, feeCalc).GetFee(nTxBytes); + CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, feeCalc).GetFee(nTxBytes); // Always obey the maximum - if (fee_needed > maxTxFee) { - fee_needed = maxTxFee; + const CAmount max_tx_fee = wallet.chain().maxTxFee(); + if (fee_needed > max_tx_fee) { + fee_needed = max_tx_fee; if (feeCalc) feeCalc->reason = FeeReason::MAXTXFEE; } return fee_needed; @@ -32,10 +32,10 @@ CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinC CFeeRate GetRequiredFeeRate(const CWallet& wallet) { - return std::max(wallet.m_min_fee, ::minRelayTxFee); + return std::max(wallet.m_min_fee, wallet.chain().relayMinFee()); } -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc) { /* User control of how to calculate fee uses the following parameter precedence: 1. coin_control.m_feerate @@ -64,7 +64,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (coin_control.m_fee_mode == FeeEstimateMode::CONSERVATIVE) conservative_estimate = true; else if (coin_control.m_fee_mode == FeeEstimateMode::ECONOMICAL) conservative_estimate = false; - feerate_needed = estimator.estimateSmartFee(target, feeCalc, conservative_estimate); + feerate_needed = wallet.chain().estimateSmartFee(target, conservative_estimate, feeCalc); if (feerate_needed == CFeeRate(0)) { // if we don't have enough data for estimateSmartFee, then use fallback fee feerate_needed = wallet.m_fallback_fee; @@ -74,7 +74,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (wallet.m_fallback_fee == CFeeRate(0)) return feerate_needed; } // Obey mempool min fee when using smart fee estimation - CFeeRate min_mempool_feerate = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + CFeeRate min_mempool_feerate = wallet.chain().mempoolMinFee(); if (feerate_needed < min_mempool_feerate) { feerate_needed = min_mempool_feerate; if (feeCalc) feeCalc->reason = FeeReason::MEMPOOL_MIN; @@ -90,13 +90,13 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr return feerate_needed; } -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator) +CFeeRate GetDiscardRate(const CWallet& wallet) { - unsigned int highest_target = estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - CFeeRate discard_rate = estimator.estimateSmartFee(highest_target, nullptr /* FeeCalculation */, false /* conservative */); + unsigned int highest_target = wallet.chain().estimateMaxBlocks(); + CFeeRate discard_rate = wallet.chain().estimateSmartFee(highest_target, false /* conservative */); // Don't let discard_rate be greater than longest possible fee estimate if we get a valid fee estimate discard_rate = (discard_rate == CFeeRate(0)) ? wallet.m_discard_rate : std::min(discard_rate, wallet.m_discard_rate); // Discard rate must be at least dustRelayFee - discard_rate = std::max(discard_rate, ::dustRelayFee); + discard_rate = std::max(discard_rate, wallet.chain().relayDustFee()); return discard_rate; } diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 6bfee456c0..434f211dc2 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -8,10 +8,8 @@ #include <amount.h> -class CBlockPolicyEstimator; class CCoinControl; class CFeeRate; -class CTxMemPool; class CWallet; struct FeeCalculation; @@ -25,7 +23,7 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes); * Estimate the minimum fee considering user set parameters * and the required fee */ -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the minimum required feerate taking into account the @@ -37,11 +35,11 @@ CFeeRate GetRequiredFeeRate(const CWallet& wallet); * Estimate the minimum fee rate considering user set parameters * and the required fee */ -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the maximum feerate for discarding change. */ -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator); +CFeeRate GetDiscardRate(const CWallet& wallet); #endif // BITCOIN_WALLET_FEES_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 20d540c8db..76a7eaa681 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -138,19 +138,22 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + return false; } else if (!fs::is_directory(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + return false; } gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); } LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); - uiInterface.InitMessage(_("Verifying wallet(s)...")); + chain.initMessage(_("Verifying wallet(s)...")); // Parameter interaction code should have thrown an error if -salvagewallet // was enabled with more than wallet file, so the wallet_files size check @@ -164,14 +167,15 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal WalletLocation location(wallet_file); if (!wallet_paths.insert(location.GetPath()).second) { - return InitError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); + chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); + return false; } std::string error_string; std::string warning_string; bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warning_string); - if (!error_string.empty()) InitError(error_string); - if (!warning_string.empty()) InitWarning(warning_string); + if (!error_string.empty()) chain.initError(error_string); + if (!warning_string.empty()) chain.initWarning(warning_string); if (!verify_success) return false; } diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp index 761e7b7dd7..1b17b09763 100644 --- a/src/wallet/psbtwallet.cpp +++ b/src/wallet/psbtwallet.cpp @@ -4,7 +4,7 @@ #include <wallet/psbtwallet.h> -bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, TransactionError& error, bool& complete, int sighash_type, bool sign, bool bip32derivs) +TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs) { LOCK(pwallet->cs_wallet); // Get all of the previous transactions @@ -19,8 +19,7 @@ bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, Transac // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. if (!input.IsSane()) { - error = TransactionError::INVALID_PSBT; - return false; + return TransactionError::INVALID_PSBT; } // If we have no utxo, grab it from the wallet. @@ -37,8 +36,7 @@ bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, Transac // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { - error = TransactionError::SIGHASH_MISMATCH; - return false; + return TransactionError::SIGHASH_MISMATCH; } complete &= SignPSBTInput(HidingSigningProvider(pwallet, !sign, !bip32derivs), psbtx, i, sighash_type); @@ -58,5 +56,5 @@ bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, Transac psbt_out.FromSignatureData(sigdata); } - return true; + return TransactionError::OK; } diff --git a/src/wallet/psbtwallet.h b/src/wallet/psbtwallet.h index b679f5c6ba..a24a0967d2 100644 --- a/src/wallet/psbtwallet.h +++ b/src/wallet/psbtwallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,16 +18,14 @@ * * @param[in] pwallet pointer to a wallet * @param[in] &psbtx reference to PartiallySignedTransaction to fill in - * @param[out] &error reference to UniValue to fill with error info on failure * @param[out] &complete indicates whether the PSBT is now complete * @param[in] sighash_type the sighash type to use when signing (if PSBT does not specify) * @param[in] sign whether to sign or not * @param[in] bip32derivs whether to fill in bip32 derivation information if available - * return true on success, false on error (and fills in `error`) + * return error */ -bool FillPSBT(const CWallet* pwallet, +NODISCARD TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, - TransactionError& error, bool& complete, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index f38202a2b8..9c5dae3623 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -157,8 +157,9 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); @@ -313,8 +314,9 @@ UniValue importaddress(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -346,7 +348,11 @@ UniValue importaddress(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -501,8 +507,9 @@ UniValue importpubkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -529,7 +536,11 @@ UniValue importpubkey(const JSONRPCRequest& request) if (fRescan) { RescanWallet(*pwallet, reserver); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } } return NullUniValue; @@ -562,8 +573,9 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.ToString()); - if (fPruneMode) + if (pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (!reserver.reserve()) { @@ -591,11 +603,11 @@ UniValue importwallet(const JSONRPCRequest& request) // Use uiInterface.ShowProgress instead of pwallet.ShowProgress because pwallet.ShowProgress has a cancel button tied to AbortRescan which // we don't want for this progress bar showing the import progress. uiInterface.ShowProgress does not have a cancel button. - uiInterface.ShowProgress(strprintf("%s " + _("Importing..."), pwallet->GetDisplayName()), 0, false); // show progress dialog in GUI + pwallet->chain().showProgress(strprintf("%s " + _("Importing..."), pwallet->GetDisplayName()), 0, false); // show progress dialog in GUI std::vector<std::tuple<CKey, int64_t, bool, std::string>> keys; std::vector<std::pair<CScript, int64_t>> scripts; while (file.good()) { - uiInterface.ShowProgress("", std::max(1, std::min(50, (int)(((double)file.tellg() / (double)nFilesize) * 100))), false); + pwallet->chain().showProgress("", std::max(1, std::min(50, (int)(((double)file.tellg() / (double)nFilesize) * 100))), false); std::string line; std::getline(file, line); if (line.empty() || line[0] == '#') @@ -633,13 +645,13 @@ UniValue importwallet(const JSONRPCRequest& request) file.close(); // We now know whether we are importing private keys, so we can error if private keys are disabled if (keys.size() > 0 && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when private keys are disabled"); } double total = (double)(keys.size() + scripts.size()); double progress = 0; for (const auto& key_tuple : keys) { - uiInterface.ShowProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); + pwallet->chain().showProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); const CKey& key = std::get<0>(key_tuple); int64_t time = std::get<1>(key_tuple); bool has_label = std::get<2>(key_tuple); @@ -664,7 +676,7 @@ UniValue importwallet(const JSONRPCRequest& request) progress++; } for (const auto& script_pair : scripts) { - uiInterface.ShowProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); + pwallet->chain().showProgress("", std::max(50, std::min(75, (int)((progress / total) * 100) + 50)), false); const CScript& script = script_pair.first; int64_t time = script_pair.second; CScriptID id(script); @@ -683,10 +695,10 @@ UniValue importwallet(const JSONRPCRequest& request) } progress++; } - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI pwallet->UpdateTimeFirstKey(nTimeBegin); } - uiInterface.ShowProgress("", 100, false); // hide progress dialog in GUI + pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI RescanWallet(*pwallet, reserver, nTimeBegin, false /* update */); pwallet->MarkDirty(); @@ -1132,13 +1144,10 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID if (!data.exists("range")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor is ranged, please specify the range"); } - const UniValue& range = data["range"]; - range_start = range.exists("start") ? range["start"].get_int64() : 0; - if (!range.exists("end")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "End of range for descriptor must be specified"); - } - range_end = range["end"].get_int64(); - if (range_end < range_start || range_start < 0) { + auto range = ParseRange(data["range"]); + range_start = range.first; + range_end = range.second; + if (range_start < 0 || (range_end >> 31) != 0 || range_end - range_start >= 1000000) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid descriptor range specified"); } } @@ -1373,12 +1382,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) {"key", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, } }, - {"range", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the start and end of the range to import", - { - {"start", RPCArg::Type::NUM, /* default */ "0", "Start of the range to import"}, - {"end", RPCArg::Type::NUM, RPCArg::Optional::NO, "End of the range to import (inclusive)"}, - } - }, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in the form [begin,end]) to import"}, {"internal", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be treated as not incoming payments (also known as change)"}, {"watchonly", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be considered watchonly."}, {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, @@ -1472,7 +1476,11 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - pwallet->ReacceptWalletTransactions(); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(*locked_chain); + } if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 37fc88dfd5..f1da1fb589 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -19,7 +19,6 @@ #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> -#include <rpc/mining.h> #include <rpc/rawtransaction.h> #include <rpc/server.h> #include <rpc/util.h> @@ -124,8 +123,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo // Add opt-in RBF status std::string rbfStatus = "no"; if (confirms <= 0) { - LOCK(mempool.cs); - RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); + RBFTransactionState rbfState = chain.isRBFOptIn(*wtx.tx); if (rbfState == RBFTransactionState::UNKNOWN) rbfStatus = "unknown"; else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) @@ -319,7 +317,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet if (nValue > curBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -341,7 +339,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet throw JSONRPCError(RPC_WALLET_ERROR, strError); } CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, state)) { strError = strprintf("Error: The transaction was rejected! Reason given: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -425,7 +423,7 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -607,7 +605,6 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -630,8 +627,9 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) @@ -679,7 +677,6 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -696,8 +693,9 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) { @@ -809,9 +807,7 @@ static UniValue sendmany(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 2 || request.params.size() > 8) - throw std::runtime_error( - RPCHelpMan{"sendmany", + const RPCHelpMan help{"sendmany", "\nSend multiple times. Amounts are double-precision floating point numbers." + HelpRequiringPassphrase(pwallet) + "\n", { @@ -821,7 +817,7 @@ static UniValue sendmany(const JSONRPCRequest& request) {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, }, }, - {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only use the balance confirmed at least this many times."}, + {"minconf", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "Ignored dummy value"}, {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment"}, {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array with addresses.\n" " The fee will be equally deducted from the amount of each selected address.\n" @@ -852,7 +848,11 @@ static UniValue sendmany(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("sendmany", "\"\", {\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\":0.01,\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\":0.02}, 6, \"testing\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -861,7 +861,7 @@ static UniValue sendmany(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -869,9 +869,6 @@ static UniValue sendmany(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Dummy value must be set to \"\""); } UniValue sendTo = request.params[1].get_obj(); - int nMinDepth = 1; - if (!request.params[2].isNull()) - nMinDepth = request.params[2].get_int(); mapValue_t mapValue; if (!request.params[3].isNull() && !request.params[3].get_str().empty()) @@ -887,7 +884,7 @@ static UniValue sendmany(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -899,7 +896,6 @@ static UniValue sendmany(const JSONRPCRequest& request) std::set<CTxDestination> destinations; std::vector<CRecipient> vecSend; - CAmount totalAmount = 0; std::vector<std::string> keys = sendTo.getKeys(); for (const std::string& name_ : keys) { CTxDestination dest = DecodeDestination(name_); @@ -916,7 +912,6 @@ static UniValue sendmany(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(sendTo[name_]); if (nAmount <= 0) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount for send"); - totalAmount += nAmount; bool fSubtractFeeFromAmount = false; for (unsigned int idx = 0; idx < subtractFeeFromAmount.size(); idx++) { @@ -931,11 +926,6 @@ static UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - // Check funds - if (totalAmount > pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth)) { - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Wallet has insufficient funds"); - } - // Shuffle recipient list std::shuffle(vecSend.begin(), vecSend.end(), FastRandomContext()); @@ -949,7 +939,7 @@ static UniValue sendmany(const JSONRPCRequest& request) if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, state)) { strFailReason = strprintf("Transaction commit failed:: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); } @@ -1051,8 +1041,6 @@ struct tallyitem static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Minimum confirmations int nMinDepth = 1; if (!params[0].isNull()) @@ -1083,8 +1071,9 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx)) { continue; + } int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < nMinDepth) @@ -2370,8 +2359,8 @@ static UniValue settxfee(const JSONRPCRequest& request) CFeeRate tx_fee_rate(nAmount, 1000); if (tx_fee_rate == 0) { // automatic selection - } else if (tx_fee_rate < ::minRelayTxFee) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", ::minRelayTxFee.ToString())); + } else if (tx_fee_rate < pwallet->chain().relayMinFee()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", pwallet->chain().relayMinFee().ToString())); } else if (tx_fee_rate < pwallet->m_min_fee) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than wallet min fee (%s)", pwallet->m_min_fee.ToString())); } @@ -2671,49 +2660,6 @@ static UniValue unloadwallet(const JSONRPCRequest& request) return NullUniValue; } -static UniValue resendwallettransactions(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( - RPCHelpMan{"resendwallettransactions", - "Immediately re-broadcast unconfirmed wallet transactions to all peers.\n" - "Intended only for testing; the wallet code periodically re-broadcasts\n" - "automatically.\n", - {}, - RPCResult{ - "Returns an RPC error if -walletbroadcast is set to false.\n" - "Returns array of transaction ids that were re-broadcast.\n" - }, - RPCExamples{""}, - }.ToString() - ); - - if (!g_connman) - throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - - auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - - if (!pwallet->GetBroadcastTransactions()) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast"); - } - - std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(*locked_chain, GetTime(), g_connman.get()); - UniValue result(UniValue::VARR); - for (const uint256& txid : txids) - { - result.push_back(txid.ToString()); - } - return result; -} - static UniValue listunspent(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -2994,7 +2940,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("feeRate")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); } - coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"]); + coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"], pwallet->chain().estimateMaxBlocks()); } if (options.exists("estimate_mode")) { if (options.exists("feeRate")) { @@ -3152,7 +3098,7 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, - {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::OMITTED, "(required for Segwit inputs) the amount spent"}, }, }, }, @@ -3284,7 +3230,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { // TODO: alias this to conf_target - coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"]); + coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"], pwallet->chain().estimateMaxBlocks()); } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); if (totalFee <= 0) { @@ -3358,62 +3304,6 @@ static UniValue bumpfee(const JSONRPCRequest& request) return result; } -UniValue generate(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"generate", - "\nMine up to nblocks blocks immediately (before the RPC call returns) to an address in the wallet.\n", - { - {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, - {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, - }, - RPCResult{ - "[ blockhashes ] (array) hashes of blocks generated\n" - }, - RPCExamples{ - "\nGenerate 11 blocks\n" - + HelpExampleCli("generate", "11") - }, - }.ToString()); - } - - if (!IsDeprecatedRPCEnabled("generate")) { - throw JSONRPCError(RPC_METHOD_DEPRECATED, "The wallet generate rpc method is deprecated and will be fully removed in v0.19. " - "To use generate in v0.18, restart bitcoind with -deprecatedrpc=generate.\n" - "Clients should transition to using the node rpc method generatetoaddress\n"); - } - - int num_generate = request.params[0].get_int(); - uint64_t max_tries = 1000000; - if (!request.params[1].isNull()) { - max_tries = request.params[1].get_int(); - } - - std::shared_ptr<CReserveScript> coinbase_script; - pwallet->GetScriptForMining(coinbase_script); - - // If the keypool is exhausted, no script is returned at all. Catch this. - if (!coinbase_script) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); - } - - //throw an error if no script was provided - if (coinbase_script->reserveScript.empty()) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "No coinbase script available"); - } - - return generateBlocks(coinbase_script, num_generate, max_tries, true); -} - UniValue rescanblockchain(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -3894,7 +3784,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }.ToString()); } - if (IsInitialBlockDownload()) { + if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); } @@ -3939,19 +3829,6 @@ UniValue sethdseed(const JSONRPCRequest& request) return NullUniValue; } -void AddKeypathToMap(const CWallet* pwallet, const CKeyID& keyID, std::map<CPubKey, KeyOriginInfo>& hd_keypaths) -{ - CPubKey vchPubKey; - if (!pwallet->GetPubKey(keyID, vchPubKey)) { - return; - } - KeyOriginInfo info; - if (!pwallet->GetKeyOrigin(keyID, info)) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Internal keypath is broken"); - } - hd_keypaths.emplace(vchPubKey, std::move(info)); -} - UniValue walletprocesspsbt(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -4007,8 +3884,8 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) bool sign = request.params[1].isNull() ? true : request.params[1].get_bool(); bool bip32derivs = request.params[3].isNull() ? false : request.params[3].get_bool(); bool complete = true; - TransactionError err; - if (!FillPSBT(pwallet, psbtx, err, complete, nHashType, sign, bip32derivs)) { + const TransactionError err = FillPSBT(pwallet, psbtx, complete, nHashType, sign, bip32derivs); + if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -4125,8 +4002,8 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? false : request.params[4].get_bool(); bool complete = true; - TransactionError err; - if (!FillPSBT(pwallet, psbtx, err, complete, 1, false, bip32derivs)) { + const TransactionError err = FillPSBT(pwallet, psbtx, complete, 1, false, bip32derivs); + if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -4156,8 +4033,6 @@ UniValue importmulti(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "generating", "generate", &generate, {"nblocks","maxtries"} }, - { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, @@ -4215,8 +4090,8 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterWalletRPCCommands(CRPCTable &t) +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) { for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + handlers.emplace_back(chain.handleRpc(commands[vcidx])); } diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 58053bde59..7cf607ccc7 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_WALLET_RPCWALLET_H #define BITCOIN_WALLET_RPCWALLET_H +#include <memory> #include <string> +#include <vector> class CRPCTable; class CWallet; @@ -14,7 +16,12 @@ class UniValue; struct PartiallySignedTransaction; class CTransaction; -void RegisterWalletRPCCommands(CRPCTable &t); +namespace interfaces { +class Chain; +class Handler; +} + +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers); /** * Figures out what wallet, if any, to use for a JSONRPCRequest. diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 2a3149de46..789e86e21b 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -62,9 +62,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) ssData >> psbtx; // Fill transaction with our data - TransactionError err; bool complete = true; - FillPSBT(&m_wallet, psbtx, err, complete, SIGHASH_ALL, false, true); + BOOST_REQUIRE_EQUAL(TransactionError::OK, FillPSBT(&m_wallet, psbtx, complete, SIGHASH_ALL, false, true)); // Get the final tx CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a5fb1db86c..6a051214a9 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -13,12 +13,7 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); - RegisterValidationInterface(&m_wallet); + m_wallet.m_chain_notifications_handler = m_chain->handleNotifications(m_wallet); - RegisterWalletRPCCommands(tableRPC); -} - -WalletTestingSetup::~WalletTestingSetup() -{ - UnregisterValidationInterface(&m_wallet); + m_chain_client->registerRpcs(); } diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index e6fe8c9473..dcfbc55f94 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -17,9 +17,9 @@ */ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - ~WalletTestingSetup(); std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); CWallet m_wallet; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index e674b2faea..af57dbf5f6 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -368,7 +368,7 @@ public: CCoinControl dummy; BOOST_CHECK(wallet->CreateTransaction(*m_locked_chain, {recipient}, tx, reservekey, fee, changePos, error, dummy)); CValidationState state; - BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, nullptr, state)); + BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, state)); CMutableTransaction blocktx; { LOCK(wallet->cs_wallet); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 388422bec8..f0499d5b32 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -12,6 +12,7 @@ #include <consensus/validation.h> #include <fs.h> #include <interfaces/chain.h> +#include <interfaces/wallet.h> #include <key.h> #include <key_io.h> #include <keystore.h> @@ -95,7 +96,7 @@ static void ReleaseWallet(CWallet* wallet) wallet->WalletLogPrintf("Releasing wallet\n"); wallet->BlockUntilSyncedToCurrentChain(); wallet->Flush(); - UnregisterValidationInterface(wallet); + wallet->m_chain_notifications_handler.reset(); delete wallet; // Wallet is now released, notify UnloadWallet, if any. { @@ -192,7 +193,7 @@ CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; @@ -280,9 +281,9 @@ void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } -bool CWallet::AddKeyPubKeyWithDB(WalletBatch &batch, const CKey& secret, const CPubKey &pubkey) +bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); // Make sure we aren't adding private keys to private key disabled wallets assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); @@ -344,16 +345,16 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, } } -void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &meta) +void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; } -void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &meta) +void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // m_script_metadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); m_script_metadata[script_id] = meta; } @@ -366,7 +367,7 @@ bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, void CWallet::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } @@ -568,7 +569,7 @@ void CWallet::ChainStateFlushed(const CBlockLocator& loc) void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool fExplicit) { - LOCK(cs_wallet); // nWalletVersion + LOCK(cs_wallet); if (nWalletVersion >= nVersion) return; @@ -592,7 +593,7 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool CWallet::SetMaxVersion(int nVersion) { - LOCK(cs_wallet); // nWalletVersion, nWalletMaxVersion + LOCK(cs_wallet); // cannot downgrade below current version if (nWalletVersion > nVersion) return false; @@ -876,9 +877,9 @@ DBErrors CWallet::ReorderTransactions() return DBErrors::LOAD_OK; } -int64_t CWallet::IncOrderPosNext(WalletBatch *batch) +int64_t CWallet::IncOrderPosNext(WalletBatch* batch) { - AssertLockHeld(cs_wallet); // nOrderPosNext + AssertLockHeld(cs_wallet); int64_t nRet = nOrderPosNext++; if (batch) { batch->WriteOrderPosNext(nOrderPosNext); @@ -940,7 +941,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.BindWallet(this); bool fInsertedNew = ret.second; if (fInsertedNew) { - wtx.nTimeReceived = GetAdjustedTime(); + wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); wtx.nTimeSmart = ComputeTimeSmart(wtx); @@ -1242,7 +1243,8 @@ void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { } } -void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { +void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) { + const uint256& block_hash = block.GetHash(); auto locked_chain = chain().lock(); LOCK(cs_wallet); // TODO: Temporarily ensure that mempool removals are notified before @@ -1257,19 +1259,19 @@ void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const SyncTransaction(ptx, {} /* block hash */, 0 /* position in block */); TransactionRemovedFromMempool(ptx); } - for (size_t i = 0; i < pblock->vtx.size(); i++) { - SyncTransaction(pblock->vtx[i], pindex->GetBlockHash(), i); - TransactionRemovedFromMempool(pblock->vtx[i]); + for (size_t i = 0; i < block.vtx.size(); i++) { + SyncTransaction(block.vtx[i], block_hash, i); + TransactionRemovedFromMempool(block.vtx[i]); } - m_last_block_processed = pindex->GetBlockHash(); + m_last_block_processed = block_hash; } -void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { +void CWallet::BlockDisconnected(const CBlock& block) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (const CTransactionRef& ptx : pblock->vtx) { + for (const CTransactionRef& ptx : block.vtx) { SyncTransaction(ptx, {} /* block hash */, 0 /* position in block */); } } @@ -1296,7 +1298,7 @@ void CWallet::BlockUntilSyncedToCurrentChain() { // ...otherwise put a callback in the validation interface queue and wait // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). - SyncWithValidationInterfaceQueue(); + chain().waitForNotifications(); } @@ -1789,7 +1791,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); } double progress_current = progress_begin; - while (block_height && !fAbortRescan && !ShutdownRequested()) { + while (block_height && !fAbortRescan && !chain().shutdownRequested()) { if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), std::max(1, std::min(99, (int)((progress_current - progress_begin) / (progress_end - progress_begin) * 100)))); } @@ -1851,7 +1853,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); result.status = ScanResult::USER_ABORT; - } else if (block_height && ShutdownRequested()) { + } else if (block_height && chain().shutdownRequested()) { WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); result.status = ScanResult::USER_ABORT; } @@ -1859,13 +1861,11 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc return result; } -void CWallet::ReacceptWalletTransactions() +void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) return; - auto locked_chain = chain().lock(); - LOCK(cs_wallet); std::map<int64_t, CWalletTx*> mapSorted; // Sort pending wallet transactions based on their initial wallet insertion order @@ -1875,7 +1875,7 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(*locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1886,25 +1886,21 @@ void CWallet::ReacceptWalletTransactions() for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); CValidationState state; - wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state); + wtx.AcceptToMemoryPool(locked_chain, state); } } -bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman) +bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain) { assert(pwallet->GetBroadcastTransactions()); if (!IsCoinBase() && !isAbandoned() && GetDepthInMainChain(locked_chain) == 0) { CValidationState state; /* GetDepthInMainChain already catches known conflicts. */ - if (InMempool() || AcceptToMemoryPool(locked_chain, maxTxFee, state)) { + if (InMempool() || AcceptToMemoryPool(locked_chain, state)) { pwallet->WalletLogPrintf("Relaying wtx %s\n", GetHash().ToString()); - if (connman) { - CInv inv(MSG_TX, GetHash()); - connman->ForEachNode([&inv](CNode* pnode) - { - pnode->PushInventory(inv); - }); + if (pwallet->chain().p2pEnabled()) { + pwallet->chain().relayTransaction(GetHash()); return true; } } @@ -2075,11 +2071,10 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Quick answer in most cases - if (!CheckFinalTx(*tx)) + if (!locked_chain.checkFinalTx(*tx)) { return false; + } int nDepth = GetDepthInMainChain(locked_chain); if (nDepth >= 1) return true; @@ -2115,53 +2110,37 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const return CTransaction(tx1) == CTransaction(tx2); } -std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman) -{ - std::vector<uint256> result; - - LOCK(cs_wallet); - - // Sort them in chronological order - std::multimap<unsigned int, CWalletTx*> mapSorted; - for (std::pair<const uint256, CWalletTx>& item : mapWallet) - { - CWalletTx& wtx = item.second; - // Don't rebroadcast if newer than nTime: - if (wtx.nTimeReceived > nTime) - continue; - mapSorted.insert(std::make_pair(wtx.nTimeReceived, &wtx)); - } - for (const std::pair<const unsigned int, CWalletTx*>& item : mapSorted) - { - CWalletTx& wtx = *item.second; - if (wtx.RelayWalletTransaction(locked_chain, connman)) - result.push_back(wtx.GetHash()); - } - return result; -} - -void CWallet::ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) +void CWallet::ResendWalletTransactions(interfaces::Chain::Lock& locked_chain, int64_t nBestBlockTime) { // Do this infrequently and randomly to avoid giving away // that these are our transactions. - if (GetTime() < nNextResend || !fBroadcastTransactions) - return; + if (GetTime() < nNextResend || !fBroadcastTransactions) return; bool fFirst = (nNextResend == 0); nNextResend = GetTime() + GetRand(30 * 60); - if (fFirst) - return; + if (fFirst) return; // Only do it if there's been a new block since last time - if (nBestBlockTime < nLastResend) - return; + if (nBestBlockTime < nLastResend) return; nLastResend = GetTime(); - // Rebroadcast unconfirmed txes older than 5 minutes before the last - // block was found: - auto locked_chain = chain().assumeLocked(); // Temporary. Removed in upcoming lock cleanup - std::vector<uint256> relayed = ResendWalletTransactionsBefore(*locked_chain, nBestBlockTime-5*60, connman); - if (!relayed.empty()) - WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed.size()); + int relayed_tx_count = 0; + + { // cs_wallet scope + LOCK(cs_wallet); + + // Relay transactions + for (std::pair<const uint256, CWalletTx>& item : mapWallet) { + CWalletTx& wtx = item.second; + // only rebroadcast unconfirmed txes older than 5 minutes before the + // last block was found + if (wtx.nTimeReceived > nBestBlockTime - 5 * 60) continue; + relayed_tx_count += wtx.RelayWalletTransaction(locked_chain) ? 1 : 0; + } + } // cs_wallet + + if (relayed_tx_count > 0) { + WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed_tx_count); + } } /** @} */ // end of mapWallet @@ -2183,9 +2162,9 @@ CAmount CWallet::GetBalance(const isminefilter& filter, const int min_depth) con LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - if (pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) >= min_depth) { - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, filter); + const CWalletTx& wtx = entry.second; + if (wtx.IsTrusted(*locked_chain) && wtx.GetDepthInMainChain(*locked_chain) >= min_depth) { + nTotal += wtx.GetAvailableCredit(*locked_chain, true, filter); } } } @@ -2201,9 +2180,9 @@ CAmount CWallet::GetUnconfirmedBalance() const LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain); + const CWalletTx& wtx = entry.second; + if (!wtx.IsTrusted(*locked_chain) && wtx.GetDepthInMainChain(*locked_chain) == 0 && wtx.InMempool()) + nTotal += wtx.GetAvailableCredit(*locked_chain); } } return nTotal; @@ -2217,8 +2196,8 @@ CAmount CWallet::GetImmatureBalance() const LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureCredit(*locked_chain); + const CWalletTx& wtx = entry.second; + nTotal += wtx.GetImmatureCredit(*locked_chain); } } return nTotal; @@ -2232,9 +2211,9 @@ CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - if (!pcoin->IsTrusted(*locked_chain) && pcoin->GetDepthInMainChain(*locked_chain) == 0 && pcoin->InMempool()) - nTotal += pcoin->GetAvailableCredit(*locked_chain, true, ISMINE_WATCH_ONLY); + const CWalletTx& wtx = entry.second; + if (!wtx.IsTrusted(*locked_chain) && wtx.GetDepthInMainChain(*locked_chain) == 0 && wtx.InMempool()) + nTotal += wtx.GetAvailableCredit(*locked_chain, true, ISMINE_WATCH_ONLY); } } return nTotal; @@ -2248,54 +2227,13 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const LOCK(cs_wallet); for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &entry.second; - nTotal += pcoin->GetImmatureWatchOnlyCredit(*locked_chain); + const CWalletTx& wtx = entry.second; + nTotal += wtx.GetImmatureWatchOnlyCredit(*locked_chain); } } return nTotal; } -// Calculate total balance in a different way from GetBalance. The biggest -// difference is that GetBalance sums up all unspent TxOuts paying to the -// wallet, while this sums up both spent and unspent TxOuts paying to the -// wallet, and then subtracts the values of TxIns spending from the wallet. This -// also has fewer restrictions on which unconfirmed transactions are considered -// trusted. -CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth) const -{ - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - - CAmount balance = 0; - for (const auto& entry : mapWallet) { - const CWalletTx& wtx = entry.second; - const int depth = wtx.GetDepthInMainChain(*locked_chain); - if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.IsImmatureCoinBase(*locked_chain)) { - continue; - } - - // Loop through tx outputs and add incoming payments. For outgoing txs, - // treat change outputs specially, as part of the amount debited. - CAmount debit = wtx.GetDebit(filter); - const bool outgoing = debit > 0; - for (const CTxOut& out : wtx.tx->vout) { - if (outgoing && IsChange(out)) { - debit -= out.nValue; - } else if (IsMine(out) & filter && depth >= minDepth) { - balance += out.nValue; - } - } - - // For outgoing txs, subtract amount debited. - if (outgoing) { - balance -= debit; - } - } - - return balance; -} - CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const { auto locked_chain = chain().lock(); @@ -2314,7 +2252,6 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t nMaximumCount, const int nMinDepth, const int nMaxDepth) const { - AssertLockHeld(cs_main); AssertLockHeld(cs_wallet); vCoins.clear(); @@ -2323,24 +2260,25 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; - const CWalletTx* pcoin = &entry.second; + const CWalletTx& wtx = entry.second; - if (!CheckFinalTx(*pcoin->tx)) + if (!locked_chain.checkFinalTx(*wtx.tx)) { continue; + } - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < 0) continue; // We should not consider coins which aren't at least in our mempool // It's possible for these to be conflicted via ancestors which we may never be able to detect - if (nDepth == 0 && !pcoin->InMempool()) + if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = pcoin->IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain); // We should not consider coins from transactions that are replacing // other transactions. @@ -2357,7 +2295,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // be a 1-block reorg away from the chain where transactions A and C // were accepted to another chain where B, B', and C were all // accepted. - if (nDepth == 0 && pcoin->mapValue.count("replaces_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaces_txid")) { safeTx = false; } @@ -2369,7 +2307,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< // intending to replace A', but potentially resulting in a scenario // where A, A', and D could all be accepted (instead of just B and // D, or just A and A' like the user would want). - if (nDepth == 0 && pcoin->mapValue.count("replaced_by_txid")) { + if (nDepth == 0 && wtx.mapValue.count("replaced_by_txid")) { safeTx = false; } @@ -2380,8 +2318,8 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth < nMinDepth || nDepth > nMaxDepth) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { - if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + if (wtx.tx->vout[i].nValue < nMinimumAmount || wtx.tx->vout[i].nValue > nMaximumAmount) continue; if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) @@ -2393,20 +2331,20 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsSpent(locked_chain, wtxid, i)) continue; - isminetype mine = IsMine(pcoin->tx->vout[i]); + isminetype mine = IsMine(wtx.tx->vout[i]); if (mine == ISMINE_NO) { continue; } - bool solvable = IsSolvable(*this, pcoin->tx->vout[i].scriptPubKey); + bool solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey); bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); - vCoins.push_back(COutput(pcoin, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { - nTotal += pcoin->tx->vout[i].nValue; + nTotal += wtx.tx->vout[i].nValue; if (nTotal >= nMinimumSumAmount) { return; @@ -2423,7 +2361,6 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Chain::Lock& locked_chain) const { - AssertLockHeld(cs_main); AssertLockHeld(cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; @@ -2488,10 +2425,10 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil FeeCalculation feeCalc; CCoinControl temp; temp.m_confirm_target = 1008; - CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this, ::feeEstimator).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); // Filter by the min conf specs and add to utxo_pool and calculate effective value for (OutputGroup& group : groups) { @@ -2565,13 +2502,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { - const CWalletTx* pcoin = &it->second; + const CWalletTx& wtx = it->second; // Clearly invalid input, fail - if (pcoin->tx->vout.size() <= outpoint.n) + if (wtx.tx->vout.size() <= outpoint.n) return false; // Just to calculate the marginal byte size - nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(CInputCoin(pcoin->tx, outpoint.n)); + nValueFromPresetInputs += wtx.tx->vout[outpoint.n].nValue; + setPresetCoins.insert(CInputCoin(wtx.tx, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -2617,9 +2554,9 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm return res; } -bool CWallet::SignTransaction(CMutableTransaction &tx) +bool CWallet::SignTransaction(CMutableTransaction& tx) { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); // sign the new tx int nIn = 0; @@ -2695,13 +2632,13 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC return true; } -static bool IsCurrentForAntiFeeSniping(interfaces::Chain::Lock& locked_chain) +static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) { - if (IsInitialBlockDownload()) { + if (chain.isInitialBlockDownload()) { return false; } constexpr int64_t MAX_ANTI_FEE_SNIPING_TIP_AGE = 8 * 60 * 60; // in seconds - if (chainActive.Tip()->GetBlockTime() < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { + if (locked_chain.getBlockTime(*locked_chain.getHeight()) < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { return false; } return true; @@ -2711,7 +2648,7 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain::Lock& locked_chain) * Return a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain::Lock& locked_chain) +static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) { uint32_t const height = locked_chain.getHeight().get_value_or(-1); uint32_t locktime; @@ -2735,7 +2672,7 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain::Lock& locked_cha // enough, that fee sniping isn't a problem yet, but by implementing a fix // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. - if (IsCurrentForAntiFeeSniping(locked_chain)) { + if (IsCurrentForAntiFeeSniping(chain, locked_chain)) { locktime = height; // Secondly occasionally randomly pick a nLockTime even further back, so @@ -2809,7 +2746,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std CMutableTransaction txNew; - txNew.nLockTime = GetLocktimeForNewTransaction(locked_chain); + txNew.nLockTime = GetLocktimeForNewTransaction(chain(), locked_chain); FeeCalculation feeCalc; CAmount nFeeNeeded; @@ -2861,10 +2798,10 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); - CFeeRate discard_rate = GetDiscardRate(*this, ::feeEstimator); + CFeeRate discard_rate = GetDiscardRate(*this); // Get the fee rate to use effective values in coin selection - CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, &feeCalc); nFeeRet = 0; bool pick_new_inputs = true; @@ -2905,7 +2842,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // Include the fee cost for outputs. Note this is only used for BnB right now coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); - if (IsDust(txout, ::dustRelayFee)) + if (IsDust(txout, chain().relayDustFee())) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2997,7 +2934,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std return false; } - nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, ::mempool, ::feeEstimator, &feeCalc); + nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, &feeCalc); if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { // eventually allow a fallback fee strFailReason = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); @@ -3006,7 +2943,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // If we made it here and we aren't even able to meet the relay fee on the next pass, give up // because we must be at the maximum allowed fee. - if (nFeeNeeded < ::minRelayTxFee.GetFee(nBytes)) + if (nFeeNeeded < chain().relayMinFee().GetFee(nBytes)) { strFailReason = _("Transaction too large for fee policy"); return false; @@ -3025,7 +2962,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // change output. Only try this once. if (nChangePosInOut == -1 && nSubtractFeeFromAmount == 0 && pick_new_inputs) { unsigned int tx_size_with_change = nBytes + coin_selection_params.change_output_size + 2; // Add 2 as a buffer in case increasing # of outputs changes compact size - CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, ::mempool, ::feeEstimator, nullptr); + CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, nullptr); CAmount minimum_value_for_change = GetDustThreshold(change_prototype_txout, discard_rate); if (nFeeRet >= fee_needed_with_change + minimum_value_for_change) { pick_new_inputs = false; @@ -3130,16 +3067,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits - LockPoints lp; - CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); - CTxMemPool::setEntries setAncestors; - size_t nLimitAncestors = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - size_t nLimitAncestorSize = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; - size_t nLimitDescendants = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; - std::string errString; - LOCK(::mempool.cs); - if (!::mempool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { + if (!chain().checkChainLimits(tx)) { strFailReason = _("Transaction has too long of a mempool chain"); return false; } @@ -3159,7 +3087,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state) +bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state) { { auto locked_chain = chain().lock(); @@ -3196,11 +3124,11 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve if (fBroadcastTransactions) { // Broadcast - if (!wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(*locked_chain, state)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", FormatStateMessage(state)); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtx.RelayWalletTransaction(*locked_chain, connman); + wtx.RelayWalletTransaction(*locked_chain); } } } @@ -3242,8 +3170,8 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - AssertLockHeld(cs_wallet); // mapWallet - DBErrors nZapSelectTxRet = WalletBatch(*database,"cr+").ZapSelectTx(vHashIn, vHashOut); + AssertLockHeld(cs_wallet); + DBErrors nZapSelectTxRet = WalletBatch(*database, "cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); @@ -3269,7 +3197,6 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 MarkDirty(); return DBErrors::LOAD_OK; - } DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) @@ -3389,7 +3316,7 @@ bool CWallet::NewKeyPool() size_t CWallet::KeypoolCountExternalKeys() { - AssertLockHeld(cs_wallet); // setExternalKeyPool + AssertLockHeld(cs_wallet); return setExternalKeyPool.size() + set_pre_split_keypool.size(); } @@ -3619,27 +3546,27 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: LOCK(cs_wallet); for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (!pcoin->IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain)) continue; - if (pcoin->IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase(locked_chain)) continue; - int nDepth = pcoin->GetDepthInMainChain(locked_chain); - if (nDepth < (pcoin->IsFromMe(ISMINE_ALL) ? 0 : 1)) + int nDepth = wtx.GetDepthInMainChain(locked_chain); + if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(pcoin->tx->vout[i])) + if (!IsMine(wtx.tx->vout[i])) continue; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, addr)) + if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : pcoin->tx->vout[i].nValue; + CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3653,19 +3580,19 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; for (const auto& walletEntry : mapWallet) { - const CWalletTx *pcoin = &walletEntry.second; + const CWalletTx& wtx = walletEntry.second; - if (pcoin->tx->vin.size() > 0) + if (wtx.tx->vin.size() > 0) { bool any_mine = false; // group all input addresses with each other - for (const CTxIn& txin : pcoin->tx->vin) + for (const CTxIn& txin : wtx.tx->vin) { CTxDestination address; if(!IsMine(txin)) /* If this input isn't mine, ignore it */ @@ -3679,7 +3606,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() // group change with input addresses if (any_mine) { - for (const CTxOut& txout : pcoin->tx->vout) + for (const CTxOut& txout : wtx.tx->vout) if (IsChange(txout)) { CTxDestination txoutAddr; @@ -3696,7 +3623,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (const auto& txout : pcoin->tx->vout) + for (const auto& txout : wtx.tx->vout) if (IsMine(txout)) { CTxDestination address; @@ -3819,38 +3746,27 @@ void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) } } -void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) -{ - std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); - CPubKey pubkey; - if (!rKey->GetReservedKey(pubkey)) - return; - - script = rKey; - script->reserveScript = CScript() << ToByteVector(pubkey) << OP_CHECKSIG; -} - void CWallet::LockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.insert(output); } void CWallet::UnlockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.erase(output); } void CWallet::UnlockAllCoins() { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.clear(); } bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); COutPoint outpt(hash, n); return (setLockedCoins.count(outpt) > 0); @@ -3858,7 +3774,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); it != setLockedCoins.end(); it++) { COutPoint outpt = (*it); @@ -3868,8 +3784,8 @@ void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const /** @} */ // end of Actions -void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t> &mapKeyBirth) const { - AssertLockHeld(cs_wallet); // mapKeyMetadata +void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t>& mapKeyBirth) const { + AssertLockHeld(cs_wallet); mapKeyBirth.clear(); // get birth times for keys with metadata @@ -4097,23 +4013,23 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, uint64_t wallet_creation_flags) { - const std::string& walletFile = location.GetName(); + const std::string& walletFile = WalletDataFilePath(location.GetPath()).string(); // needed to restore wallet transaction meta data after -zapwallettxes std::vector<CWalletTx> vWtx; if (gArgs.GetBoolArg("-zapwallettxes", false)) { - uiInterface.InitMessage(_("Zapping all transactions from wallet...")); + chain.initMessage(_("Zapping all transactions from wallet...")); std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(chain, location, WalletDatabase::Create(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } } - uiInterface.InitMessage(_("Loading wallet...")); + chain.initMessage(_("Loading wallet...")); int64_t nStart = GetTimeMillis(); bool fFirstRun = true; @@ -4124,31 +4040,31 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { - InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect."), + chain.initWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - InitError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); + chain.initError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - InitError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); + chain.initError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); return nullptr; } else { - InitError(strprintf(_("Error loading %s"), walletFile)); + chain.initError(strprintf(_("Error loading %s"), walletFile)); return nullptr; } } - int prev_version = walletInstance->nWalletVersion; + int prev_version = walletInstance->GetVersion(); if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) { int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); @@ -4162,7 +4078,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); if (nMaxVersion < walletInstance->GetVersion()) { - InitError(_("Cannot downgrade wallet")); + chain.initError(_("Cannot downgrade wallet")); return nullptr; } walletInstance->SetMaxVersion(nMaxVersion); @@ -4173,9 +4089,9 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, LOCK(walletInstance->cs_wallet); // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->nWalletVersion; - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >=FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - InitError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); + int max_version = walletInstance->GetVersion(); + if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + chain.initError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); return nullptr; } @@ -4203,7 +4119,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Regenerate the keypool if upgraded to HD if (hd_upgrade) { if (!walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate keys")); + chain.initError(_("Unable to generate keys")); return nullptr; } } @@ -4227,7 +4143,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Top up the keypool if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate initial keys")); + chain.initError(_("Unable to generate initial keys")); return nullptr; } @@ -4235,34 +4151,34 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->ChainStateFlushed(locked_chain->getTipLocator()); } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation - InitError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { LOCK(walletInstance->cs_KeyStore); if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - InitWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); + chain.initWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); } } if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - InitError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); + chain.initError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); return nullptr; } if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - InitError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); + chain.initError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); return nullptr; } if (gArgs.IsArgSet("-mintxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - InitError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); + chain.initError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); return nullptr; } if (n > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-mintxfee") + " " + - _("This is the minimum transaction fee you pay on every transaction.")); + chain.initWarning(AmountHighWarn("-mintxfee") + " " + + _("This is the minimum transaction fee you pay on every transaction.")); } walletInstance->m_min_fee = CFeeRate(n); } @@ -4271,12 +4187,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); + chain.initError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-fallbackfee") + " " + - _("This is the transaction fee you may pay when fee estimates are not available.")); + chain.initWarning(AmountHighWarn("-fallbackfee") + " " + + _("This is the transaction fee you may pay when fee estimates are not available.")); } walletInstance->m_fallback_fee = CFeeRate(nFeePerK); walletInstance->m_allow_fallback_fee = nFeePerK != 0; //disable fallback fee in case value was set to 0, enable if non-null value @@ -4284,29 +4200,29 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-discardfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); + chain.initError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-discardfee") + " " + - _("This is the transaction fee you may discard if change is smaller than dust at this level")); + chain.initWarning(AmountHighWarn("-discardfee") + " " + + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } walletInstance->m_discard_rate = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - InitError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); + chain.initError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-paytxfee") + " " + - _("This is the transaction fee you will pay if you send a transaction.")); + chain.initWarning(AmountHighWarn("-paytxfee") + " " + + _("This is the transaction fee you will pay if you send a transaction.")); } walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); - if (walletInstance->m_pay_tx_fee < ::minRelayTxFee) { - InitError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), - gArgs.GetArg("-paytxfee", ""), ::minRelayTxFee.ToString())); + if (walletInstance->m_pay_tx_fee < chain.relayMinFee()) { + chain.initError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), + gArgs.GetArg("-paytxfee", ""), chain.relayMinFee().ToString())); return nullptr; } } @@ -4346,20 +4262,19 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, //We can't rescan beyond non-pruned blocks, stop and throw an error //this might happen if a user uses an old wallet within a pruned node // or if he ran -disablewallet for a longer time, then decided to re-enable - if (fPruneMode) - { + if (chain.getPruneMode()) { int block_height = *tip_height; while (block_height > 0 && locked_chain->haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - InitError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); + chain.initError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); return nullptr; } } - uiInterface.InitMessage(_("Rescanning...")); + chain.initMessage(_("Rescanning...")); walletInstance->WalletLogPrintf("Rescanning last %i blocks (from block %i)...\n", *tip_height - rescan_height, rescan_height); // No need to read and scan block if block was created before @@ -4374,7 +4289,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, { WalletRescanReserver reserver(walletInstance.get()); if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { - InitError(_("Failed to rescan the wallet during initialization")); + chain.initError(_("Failed to rescan the wallet during initialization")); return nullptr; } } @@ -4407,10 +4322,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - uiInterface.LoadWallet(walletInstance); + chain.loadWallet(interfaces::MakeWallet(walletInstance)); - // Register with the validation interface. It's ok to do this after rescan since we're still holding cs_main. - RegisterValidationInterface(walletInstance.get()); + // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. + walletInstance->m_chain_notifications_handler = chain.handleNotifications(*walletInstance); walletInstance->SetBroadcastTransactions(gArgs.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); @@ -4425,9 +4340,15 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, void CWallet::postInitProcess() { + auto locked_chain = chain().lock(); + LOCK(cs_wallet); + // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(); + ReacceptWalletTransactions(*locked_chain); + + // Update wallet transactions with current mempool transactions. + chain().requestMempoolTransactions(*this); } bool CWallet::BackupWallet(const std::string& strDest) @@ -4470,8 +4391,6 @@ int CMerkleTx::GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const if (hashUnset()) return 0; - AssertLockHeld(cs_main); - return locked_chain.getBlockDepth(hashBlock) * (nIndex == -1 ? -1 : 1); } @@ -4490,17 +4409,14 @@ bool CMerkleTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const return GetBlocksToMaturity(locked_chain) > 0; } -bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state) { - LockAnnotation lock(::cs_main); // Temporary, for AcceptToMemoryPool below. Removed in upcoming commit. - // We must set fInMempool here - while it will be re-set to true by the // entered-mempool callback, if we did not there would be a race where a // user could call sendmoney in a loop and hit spurious out of funds errors // because we think that this newly generated transaction's change is // unavailable as we're not yet aware that it is in the mempool. - bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, - nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + bool ret = locked_chain.submitToMemoryPool(tx, pwallet->chain().maxTxFee(), state); fInMempool |= ret; return ret; } @@ -4531,7 +4447,7 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; - mempool.GetTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { // Limit output groups to no more than 10 entries, to protect // against inadvertently creating a too-large transaction diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 2a5d6caaf8..3aeeaafd1f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -8,6 +8,7 @@ #include <amount.h> #include <interfaces/chain.h> +#include <interfaces/handler.h> #include <outputtype.h> #include <policy/feerate.h> #include <streams.h> @@ -100,8 +101,6 @@ class CCoinControl; class COutput; class CReserveKey; class CScript; -class CTxMemPool; -class CBlockPolicyEstimator; class CWalletTx; struct FeeCalculation; enum class FeeEstimateMode; @@ -507,7 +506,7 @@ public: CAmount GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const; CAmount GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true) const; // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct - // annotation "EXCLUSIVE_LOCKS_REQUIRED(cs_main, pwallet->cs_wallet)". The + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid // having to resolve the issue of member access into incomplete type CWallet. CAmount GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true, const isminefilter& filter=ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; @@ -537,10 +536,10 @@ public: int64_t GetTxTime() const; // RelayWalletTransaction may only be called if fBroadcastTransactions! - bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman); + bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain); /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state); + bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -639,7 +638,7 @@ class WalletRescanReserver; //forward declarations for ScanForWalletTransactions * A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, * and provides the ability to create new transactions. */ -class CWallet final : public CCryptoKeyStore, public CValidationInterface +class CWallet final : public CCryptoKeyStore, private interfaces::Chain::Notifications { private: std::atomic<bool> fAbortRescan{false}; @@ -650,7 +649,7 @@ private: WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; //! the current wallet version: clients below this version are not able to load the wallet - int nWalletVersion = FEATURE_BASE; + int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; //! the maximum wallet format version: memory-only variable that specifies to what version this wallet may be upgraded int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; @@ -700,11 +699,11 @@ private: CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::set<int64_t> setInternalKeyPool; + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool; + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; std::atomic<uint64_t> m_wallet_flags{0}; @@ -810,6 +809,9 @@ public: std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); + /** Registered interfaces::Chain::Notifications handler. */ + std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; + /** Interface for accessing chain state. */ interfaces::Chain& chain() const { return m_chain; } @@ -922,8 +924,8 @@ public: bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); void LoadToWallet(const CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void TransactionAddedToMempool(const CTransactionRef& tx) override; - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const CBlock& block) override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); struct ScanResult { @@ -943,16 +945,13 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(); - void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override EXCLUSIVE_LOCKS_REQUIRED(cs_main); - // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! - std::vector<uint256> ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman); + void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ResendWalletTransactions(interfaces::Chain::Lock& locked_chain, int64_t nBestBlockTime) override; CAmount GetBalance(const isminefilter& filter=ISMINE_SPENDABLE, const int min_depth=0) const; CAmount GetUnconfirmedBalance() const; CAmount GetImmatureBalance() const; CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; - CAmount GetLegacyBalance(const isminefilter& filter, int minDepth) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); @@ -971,7 +970,7 @@ public: */ bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); - bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state); + bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state); bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const { @@ -1065,11 +1064,9 @@ public: const std::string& GetLabelName(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void GetScriptForMining(std::shared_ptr<CReserveScript> &script); - unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { - AssertLockHeld(cs_wallet); // set{Ex,In}ternalKeyPool + AssertLockHeld(cs_wallet); return setInternalKeyPool.size() + setExternalKeyPool.size(); } @@ -1224,6 +1221,8 @@ public: /** Add a KeyOriginInfo to the wallet */ bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info); + + friend struct WalletTestingSetup; }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 2783f83fd6..5149bb0263 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -422,8 +422,15 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strType != "minversion" && strType != "acentry") { wss.m_unknown_records++; } - } catch (...) - { + } catch (const std::exception& e) { + if (strErr.empty()) { + strErr = e.what(); + } + return false; + } catch (...) { + if (strErr.empty()) { + strErr = "Caught unknown exception in ReadKeyValue"; + } return false; } return true; diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 6db4c63acb..2d8adf8ba6 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -4,6 +4,7 @@ #include <wallet/walletutil.h> +#include <logging.h> #include <util/system.h> fs::path GetWalletDir() @@ -33,9 +34,11 @@ static bool IsBerkeleyBtree(const fs::path& path) // A Berkeley DB Btree file has at least 4K. // This check also prevents opening lock files. boost::system::error_code ec; - if (fs::file_size(path, ec) < 4096) return false; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 4096) return false; - fs::ifstream file(path.string(), std::ios::binary); + fsbridge::ifstream file(path, std::ios::binary); if (!file.is_open()) return false; file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 @@ -54,8 +57,14 @@ std::vector<fs::path> ListWalletDir() const fs::path wallet_dir = GetWalletDir(); const size_t offset = wallet_dir.string().size() + 1; std::vector<fs::path> paths; + boost::system::error_code ec; + + for (auto it = fs::recursive_directory_iterator(wallet_dir, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (ec) { + LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); + continue; + } - for (auto it = fs::recursive_directory_iterator(wallet_dir); it != fs::recursive_directory_iterator(); ++it) { // Get wallet path relative to walletdir by removing walletdir from the wallet path. // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. const fs::path path = it->path().string().substr(offset); diff --git a/test/functional/example_test.py b/test/functional/example_test.py index f367e4fca8..0e70ebba6d 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 12a4ce9aff..e7e4f84ad9 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -180,7 +180,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(150) + p2p1.sync_with_ping(200) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8466f851ca..21c832c1a9 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -10,7 +10,7 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, ToHex from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, get_bip9_status, satoshi_round, sync_blocks +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, get_bip9_status, satoshi_round, sync_blocks SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -63,7 +63,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC utxos = self.nodes[0].listunspent(0, 0) - assert(len(utxos) > 0) + assert len(utxos) > 0 utxo = utxos[0] @@ -253,7 +253,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].generate(1) cur_time += 600 - assert(tx2.hash in self.nodes[0].getrawmempool()) + assert tx2.hash in self.nodes[0].getrawmempool() test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=True) test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) @@ -264,23 +264,23 @@ class BIP68Test(BitcoinTestFramework): # Advance the time on the node so that we can test timelocks self.nodes[0].setmocktime(cur_time+600) self.nodes[0].generate(1) - assert(tx2.hash not in self.nodes[0].getrawmempool()) + assert tx2.hash not in self.nodes[0].getrawmempool() # Now that tx2 is not in the mempool, a sequence locked spend should # succeed tx3 = test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx3.hash in self.nodes[0].getrawmempool() self.nodes[0].generate(1) - assert(tx3.hash not in self.nodes[0].getrawmempool()) + assert tx3.hash not in self.nodes[0].getrawmempool() # One more test, this time using height locks tx4 = test_nonzero_locks(tx3, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx4.hash in self.nodes[0].getrawmempool()) + assert tx4.hash in self.nodes[0].getrawmempool() # Now try combining confirmed and unconfirmed inputs tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx5.hash not in self.nodes[0].getrawmempool()) + assert tx5.hash not in self.nodes[0].getrawmempool() utxos = self.nodes[0].listunspent() tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) @@ -299,8 +299,8 @@ class BIP68Test(BitcoinTestFramework): # If we invalidate the tip, tx3 should get added to the mempool, causing # tx4 to be removed (fails sequence-lock). self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - assert(tx4.hash not in self.nodes[0].getrawmempool()) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx4.hash not in self.nodes[0].getrawmempool() + assert tx3.hash in self.nodes[0].getrawmempool() # Now mine 2 empty blocks to reorg out the current tip (labeled tip-1 in # diagram above). @@ -319,8 +319,8 @@ class BIP68Test(BitcoinTestFramework): cur_time += 1 mempool = self.nodes[0].getrawmempool() - assert(tx3.hash not in mempool) - assert(tx2.hash in mempool) + assert tx3.hash not in mempool + assert tx2.hash in mempool # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) @@ -332,7 +332,7 @@ class BIP68Test(BitcoinTestFramework): # being run, then it's possible the test has activated the soft fork, and # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): - assert(get_bip9_status(self.nodes[0], 'csv')['status'] != 'active') + assert get_bip9_status(self.nodes[0], 'csv')['status'] != 'active' txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) tx1 = FromHex(CTransaction(), self.nodes[0].getrawtransaction(txid)) @@ -372,7 +372,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 5253ff7aaa..faf7f20257 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -773,7 +773,7 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(57) b58 = self.next_block(58, spend=out[17]) tx = CTransaction() - assert(len(out[17].vout) < 42) + assert len(out[17].vout) < 42 tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), 0xffffffff)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() @@ -824,7 +824,7 @@ class FullBlockTest(BitcoinTestFramework): tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) - assert(tx.vin[0].nSequence < 0xffffffff) + assert tx.vin[0].nSequence < 0xffffffff tx.calc_sha256() b62 = self.update_block(62, [tx]) self.sync_blocks([b62], success=False, reject_reason='bad-txns-nonfinal') @@ -1143,8 +1143,8 @@ class FullBlockTest(BitcoinTestFramework): # now check that tx78 and tx79 have been put back into the peer's mempool mempool = self.nodes[0].getrawmempool() assert_equal(len(mempool), 2) - assert(tx78.hash in mempool) - assert(tx79.hash in mempool) + assert tx78.hash in mempool + assert tx79.hash in mempool # Test invalid opcodes in dead execution paths. # diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 302a5ec1cb..b16eafccca 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -15,7 +15,6 @@ from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, O from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, hex_str_to_bytes, ) @@ -60,6 +59,7 @@ class BIP65Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -113,7 +113,7 @@ class BIP65Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 4b3f6603a2..13481b0478 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -41,13 +41,21 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided') + inc_conf_file2_path = os.path.join(self.nodes[0].datadir, 'include2.conf') + with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: + conf.write('includeconf={}\n'.format(inc_conf_file2_path)) + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('testnot.datadir=1\n[testnet]\n') + conf.write('testnot.datadir=1\n') + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('') # clear def run_test(self): self.stop_node(0) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index df79c4312c..ecc68217bb 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test activation of the first version bits soft fork. @@ -106,7 +106,7 @@ def send_generic_input_tx(node, coinbases, address): def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip68inputs) >= 16) + assert len(bip68inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip68inputs[i], address, amount=Decimal("49.98")) @@ -121,7 +121,7 @@ def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): def create_bip112txs(node, bip112inputs, varyOP_CSV, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip112inputs) >= 16) + assert len(bip112inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip112inputs[i], address, amount=Decimal("49.98")) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 8b06cc7372..62062926a6 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. @@ -28,18 +28,12 @@ import errno import http.client import random -import sys import time from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, ToHex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, create_confirmed_utxos, hex_str_to_bytes -HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest] -try: - HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected) -except AttributeError: - pass class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): @@ -110,14 +104,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): try: self.nodes[node_index].submitblock(block) return True - except http.client.BadStatusLine as e: - # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error. - if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''": - self.log.debug("node %d submitblock raised exception: %s", node_index, e) - return False - else: - raise - except tuple(HTTP_DISCONNECT_ERRORS) as e: + except (http.client.CannotSendRequest, http.client.RemoteDisconnected) as e: self.log.debug("node %d submitblock raised exception: %s", node_index, e) return False except OSError as e: diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 9cbc1b39bd..7480e5c5ba 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -14,7 +14,6 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, wait_until, ) @@ -47,6 +46,7 @@ class BIP66Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1', '-enablebip61']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -102,7 +102,7 @@ class BIP66Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index b68e46adbc..28c15c269b 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -65,7 +65,7 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee # the ScriptSig that will satisfy the ScriptPubKey. for inp in tx.vin: inp.scriptSig = SCRIPT_SIG[inp.prevout.n] - txid = from_node.sendrawtransaction(ToHex(tx), True) + txid = from_node.sendrawtransaction(hexstring=ToHex(tx), maxfeerate=0) unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) unconflist.append({"txid": txid, "vout": 1, "amount": amount}) @@ -95,7 +95,7 @@ def split_inputs(from_node, txins, txouts, initial_split=False): else: tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] completetx = ToHex(tx) - txid = from_node.sendrawtransaction(completetx, True) + txid = from_node.sendrawtransaction(hexstring=completetx, maxfeerate=0) txouts.append({"txid": txid, "vout": 0, "amount": half_change}) txouts.append({"txid": txid, "vout": 1, "amount": rem_change}) diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index d8083b2840..13cf951550 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" @@ -66,23 +66,7 @@ class NotificationsTest(BitcoinTestFramework): txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) assert_equal(sorted(txids_rpc), sorted(os.listdir(self.walletnotify_dir))) - # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. - self.log.info("test -alertnotify") - self.nodes[1].generatetoaddress(41, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - # Give bitcoind 10 seconds to write the alert notification - wait_until(lambda: len(os.listdir(self.alertnotify_dir)), timeout=10) - - for notify_file in os.listdir(self.alertnotify_dir): - os.remove(os.path.join(self.alertnotify_dir, notify_file)) - - # Mine more up-version blocks, should not get more alerts: - self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - self.log.info("-alertnotify should not continue notifying for more unknown version blocks") - assert_equal(len(os.listdir(self.alertnotify_dir)), 0) + # TODO: add test for `-alertnotify` large fork notifications if __name__ == '__main__': NotificationsTest().main() diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index eb76089d9c..a56c983ccc 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -18,7 +18,7 @@ from test_framework.blocktools import create_coinbase, create_block, create_tran from test_framework.messages import CTransaction from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str +from test_framework.util import assert_equal, assert_raises_rpc_error NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" @@ -27,7 +27,7 @@ def trueDummy(tx): newscript = [] for i in scriptSig: if (len(newscript) == 0): - assert(len(i) == 0) + assert len(i) == 0 newscript.append(b'\x51') else: newscript.append(i) @@ -64,17 +64,17 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] - txid1 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[0].serialize_with_witness()), True) + txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) - txid2 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[1].serialize_with_witness()), True) + txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) - txid3 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[2].serialize_with_witness()), True) + txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test1txs, False, True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) trueDummy(test2tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], False, True) @@ -83,19 +83,19 @@ class NULLDUMMYTest(BitcoinTestFramework): test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test5tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test5tx], True) self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") for i in test6txs: - self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) + self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test6txs, True, True) def block_submit(self, node, txs, witness=False, accept=False): @@ -108,7 +108,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 31d2ee8e13..be323d355e 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. @@ -44,6 +44,7 @@ RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports class ProxyTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + self.setup_clean_chain = True def setup_nodes(self): self.have_ipv6 = test_ipv6_local() @@ -94,7 +95,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv4 connection through node node.addnode("15.61.23.23:1234", "onetry") cmd = proxies[0].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"15.61.23.23") @@ -108,7 +109,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv6 connection through node node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"1233:3432:2434:2343:3234:2345:6546:4534") @@ -122,7 +123,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing onion connection through node node.addnode("bitcoinostk4e4re.onion:8333", "onetry") cmd = proxies[2].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"bitcoinostk4e4re.onion") assert_equal(cmd.port, 8333) @@ -134,7 +135,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing DNS name connection through node node.addnode("node.noumenon:8333", "onetry") cmd = proxies[3].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"node.noumenon") assert_equal(cmd.port, 8333) @@ -198,4 +199,3 @@ class ProxyTest(BitcoinTestFramework): if __name__ == '__main__': ProxyTest().main() - diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 3e1ba88f0a..49951e24f3 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -8,11 +8,13 @@ WARNING: This test uses 4GB of disk space. This test takes 30 mins or more (up to 2 hours) """ +import os +from test_framework.blocktools import create_coinbase +from test_framework.messages import CBlock, ToHex +from test_framework.script import CScript, OP_RETURN, OP_NOP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until - -import os +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, wait_until MIN_BLOCKS_TO_KEEP = 288 @@ -21,19 +23,59 @@ MIN_BLOCKS_TO_KEEP = 288 # compatible with pruning based on key creation time. TIMESTAMP_WINDOW = 2 * 60 * 60 +def mine_large_blocks(node, n): + # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN + # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase + # transaction but is consensus valid. + + # Get the block parameters for the first block + big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) + best_block = node.getblock(node.getbestblockhash()) + height = int(best_block["height"]) + 1 + try: + # Static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1 + except AttributeError: + mine_large_blocks.nTime = int(best_block["time"]) + 1 + previousblockhash = int(best_block["hash"], 16) + + for _ in range(n): + # Build the coinbase transaction (with large scriptPubKey) + coinbase_tx = create_coinbase(height) + coinbase_tx.vin[0].nSequence = 2 ** 32 - 1 + coinbase_tx.vout[0].scriptPubKey = big_script + coinbase_tx.rehash() + + # Build the block + block = CBlock() + block.nVersion = best_block["version"] + block.hashPrevBlock = previousblockhash + block.nTime = mine_large_blocks.nTime + block.nBits = int('207fffff', 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Submit to the node + node.submitblock(ToHex(block)) + + previousblockhash = block.sha256 + height += 1 + mine_large_blocks.nTime += 1 def calc_usage(blockdir): - return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) + return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timeout = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ @@ -55,7 +97,7 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[0], 4) sync_blocks(self.nodes[0:5]) @@ -71,21 +113,19 @@ class PruneTest(BitcoinTestFramework): self.nodes[1].generate(200) sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MiB of data - for i in range(645): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 645) sync_blocks(self.nodes[0:5]) def test_height_min(self): - if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): - raise AssertionError("blk00000.dat is missing, pruning too early") + assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) @@ -93,8 +133,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") + assert_greater_than(550, usage) def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks @@ -103,26 +142,17 @@ class PruneTest(BitcoinTestFramework): for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects - # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine - self.stop_node(0) - self.start_node(0, extra_args=self.full_node_default_args) + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[0], 2) # Mine 24 blocks in node 1 - for i in range(24): - if j == 0: - mine_large_block(self.nodes[1], self.utxo_cache_1) - else: - # Add node1's wallet transactions back to the mempool, to - # avoid the mined blocks from being too small. - self.nodes[1].resendwallettransactions() - self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + mine_large_blocks(self.nodes[1], 24) # Reorg back with 25 block chain from node 0 - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Create connections in the order so both nodes can see the reorg at the same time - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) sync_blocks(self.nodes[0:3]) self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) @@ -130,63 +160,48 @@ class PruneTest(BitcoinTestFramework): def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain - # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) - # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) - invalidheight = height-287 - badhash = self.nodes[1].getblockhash(invalidheight) - self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) - self.nodes[1].invalidateblock(badhash) + self.forkheight = height - 287 + self.forkhash = self.nodes[1].getblockhash(self.forkheight) + self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) - mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) while curhash != mainchainhash: self.nodes[1].invalidateblock(curhash) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) - assert(self.nodes[1].getblockcount() == invalidheight - 1) + assert self.nodes[1].getblockcount() == self.forkheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) - # Reboot node1 to clear those giant tx's from mempool - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) + # Disconnect node1 and generate the new chain + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[1], 2) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[2], 1) + connect_nodes(self.nodes[1], 2) sync_blocks(self.nodes[0:3], timeout=120) self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) - - self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) - # Get node0's wallet transactions back in its mempool, to avoid the - # mined blocks from being too small. - self.nodes[0].resendwallettransactions() + self.log.info("Mine 220 more large blocks so we have requisite history") - for i in range(22): - # This can be slow, so do this in multiple RPC calls to avoid - # RPC timeouts. - self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects - sync_blocks(self.nodes[0:3], timeout=300) + mine_large_blocks(self.nodes[0], 220) usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") - - return invalidheight,badhash + assert_greater_than(550, usage) def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away @@ -219,17 +234,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert(self.nodes[0].getblockcount() == self.mainchainheight) - assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) + assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert(self.nodes[2].getbestblockhash() == goalbesthash) + assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned - assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode @@ -287,38 +302,30 @@ class PruneTest(BitcoinTestFramework): # height=100 too low to prune first block file so this is a no-op prune(100) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # Does nothing node.pruneblockchain(height(0)) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # height=500 should prune first file prune(500) - if has_block(0): - raise AssertionError("blk00000.dat is still there, should be pruned by now") - if not has_block(1): - raise AssertionError("blk00001.dat is missing when should still be there") + assert not has_block(0), "blk00000.dat is still there, should be pruned by now" + assert has_block(1), "blk00001.dat is missing when should still be there" # height=650 should prune second file prune(650) - if has_block(1): - raise AssertionError("blk00001.dat is still there, should be pruned by now") + assert not has_block(1), "blk00001.dat is still there, should be pruned by now" # height=1000 should not prune anything more, because tip-288 is in blk00002.dat. prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) - if not has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") + assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) node.generate(288) prune(1000) - if has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") - if has_block(3): - raise AssertionError("blk00003.dat is still there, should be pruned by now") + assert not has_block(2), "blk00002.dat is still there, should be pruned by now" + assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) @@ -339,21 +346,14 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) #stop and start to trigger rescan + self.stop_node(5) # stop and start to trigger rescan self.start_node(5, extra_args=["-prune=550"]) self.log.info("Success") def run_test(self): - self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - self.log.info("Mining a big blockchain of 995 blocks") - - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] + self.log.info("Warning! This test requires 4GB of disk space") + self.log.info("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -394,11 +394,11 @@ class PruneTest(BitcoinTestFramework): # +...+(1044) &.. $...$(1319) # Save some current chain state for later use - self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.log.info("Check that we can survive a 288 block reorg still") - (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + self.reorg_test() # (1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index ff7c2b23bf..ccba547a1c 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -9,12 +9,12 @@ from decimal import Decimal from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, satoshi_round +from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round MAX_REPLACEMENT_LIMIT = 100 def txToHex(tx): - return bytes_to_hex_str(tx.serialize()) + return tx.serialize().hex() def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): """Create a txout with a given amount and scriptPubKey @@ -46,7 +46,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): signed_tx = node.signrawtransactionwithwallet(txToHex(tx2)) - txid = node.sendrawtransaction(signed_tx['hex'], True) + txid = node.sendrawtransaction(signed_tx['hex'], 0) # If requested, ensure txouts are confirmed. if confirmed: @@ -56,7 +56,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): new_size = len(node.getrawmempool()) # Error out if we have something stuck in the mempool, as this # would likely be a bug. - assert(new_size < mempool_size) + assert new_size < mempool_size mempool_size = new_size return COutPoint(int(txid, 16), 0) @@ -136,7 +136,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) self.sync_all() @@ -147,9 +147,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # This will raise an exception due to transaction replacement being disabled - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Extra 0.1 BTC fee tx1b = CTransaction() @@ -157,14 +157,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(int(0.9 * COIN), CScript([b'b' * 35]))] tx1b_hex = txToHex(tx1b) # Replacement still disabled even with "enough fee" - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Works when enabled - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) mempool = self.nodes[0].getrawmempool() - assert (tx1a_txid not in mempool) - assert (tx1b_txid in mempool) + assert tx1a_txid not in mempool + assert tx1b_txid in mempool assert_equal(tx1b_hex, self.nodes[0].getrawtransaction(tx1b_txid)) @@ -188,7 +188,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vin = [CTxIn(prevout, nSequence=0)] tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] tx_hex = txToHex(tx) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) chain_txids.append(txid) prevout = COutPoint(int(txid, 16), 0) @@ -200,18 +200,18 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for doublespent_txid in chain_txids: - assert(doublespent_txid not in mempool) + assert doublespent_txid not in mempool def test_doublespend_tree(self): """Doublespend of a big tree of transactions""" @@ -236,8 +236,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vout = vout tx_hex = txToHex(tx) - assert(len(tx.serialize()) < 100000) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + assert len(tx.serialize()) < 100000 + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) yield tx _total_txs[0] += 1 @@ -261,20 +261,20 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 1 BTC fee is enough dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(initial_nValue - fee * n - 1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for tx in tree_txs: tx.rehash() - assert (tx.hash not in mempool) + assert tx.hash not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -289,7 +289,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) for tx in tree_txs: tx.rehash() @@ -303,7 +303,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - self.nodes[0].sendrawtransaction(tx1a_hex, True) + self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. @@ -313,7 +313,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) def test_spends_of_conflicting_outputs(self): """Replacements that spend conflicting tx outputs are rejected""" @@ -324,7 +324,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(utxo1, nSequence=0)] tx1a.vout = [CTxOut(int(1.1 * COIN), CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) tx1a_txid = int(tx1a_txid, 16) @@ -336,14 +336,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. tx1b = CTransaction() tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] tx1b.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1b_hex = txToHex(tx1b) - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) tx1b_txid = int(tx1b_txid, 16) tx2 = CTransaction() @@ -353,7 +353,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" @@ -364,7 +364,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1.vin = [CTxIn(confirmed_utxo)] tx1.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1_hex = txToHex(tx1) - self.nodes[0].sendrawtransaction(tx1_hex, True) + self.nodes[0].sendrawtransaction(tx1_hex, 0) tx2 = CTransaction() tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] @@ -372,7 +372,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_too_many_replacements(self): """Replacements that evict too many transactions are rejected""" @@ -394,7 +394,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): splitting_tx.vout = outputs splitting_tx_hex = txToHex(splitting_tx) - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, True) + txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) txid = int(txid, 16) # Now spend each of those outputs individually @@ -403,7 +403,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] tx_i.vout = [CTxOut(split_value - fee, CScript([b'a' * 35]))] tx_i_hex = txToHex(tx_i) - self.nodes[0].sendrawtransaction(tx_i_hex, True) + self.nodes[0].sendrawtransaction(tx_i_hex, 0) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have @@ -418,14 +418,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx_hex = txToHex(double_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass double_tx = CTransaction() double_tx.vin = inputs[0:-1] double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] double_tx_hex = txToHex(double_tx) - self.nodes[0].sendrawtransaction(double_tx_hex, True) + self.nodes[0].sendrawtransaction(double_tx_hex, 0) def test_opt_in(self): """Replacing should only work if orig tx opted in""" @@ -436,7 +436,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0xffffffff)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # This transaction isn't shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) @@ -448,7 +448,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -457,7 +457,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, True) + tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Still shouldn't be able to double-spend tx2b = CTransaction() @@ -466,7 +466,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now create a new transaction that spends from tx1a and tx2a # opt-in on one of the inputs @@ -481,7 +481,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3a.vout = [CTxOut(int(0.9*COIN), CScript([b'c'])), CTxOut(int(0.9*COIN), CScript([b'd']))] tx3a_hex = txToHex(tx3a) - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, True) + tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) @@ -496,10 +496,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3c.vout = [CTxOut(int(0.5 * COIN), CScript([b'f' * 35]))] tx3c_hex = txToHex(tx3c) - self.nodes[0].sendrawtransaction(tx3b_hex, True) + self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, True) + self.nodes[0].sendrawtransaction(tx3c_hex, 0) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -512,7 +512,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the actual fee per KB is much lower. tx1b = CTransaction() @@ -521,15 +521,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # Verify tx1b cannot replace tx1a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # Use prioritisetransaction to set tx1a's fee to 0. self.nodes[0].prioritisetransaction(txid=tx1a_txid, fee_delta=int(-0.1*COIN)) # Now tx1b should be able to replace tx1a - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - assert(tx1b_txid in self.nodes[0].getrawmempool()) + assert tx1b_txid in self.nodes[0].getrawmempool() # 2. Check that absolute fee checks use modified fee. tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -538,7 +538,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - self.nodes[0].sendrawtransaction(tx2a_hex, True) + self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Lower fee, but we'll prioritise it tx2b = CTransaction() @@ -548,15 +548,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now prioritise tx2b to have a higher modified fee self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1*COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) - assert(tx2b_txid in self.nodes[0].getrawmempool()) + assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): us0 = self.nodes[0].listunspent()[0] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 658a8cd75e..226a7dc633 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -18,7 +18,7 @@ from test_framework.blocktools import witness_script, send_to_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, sha256, ToHex from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc NODE_0 = 0 NODE_2 = 2 @@ -93,17 +93,17 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -172,16 +172,16 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False)) - assert(self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False)) + assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) + assert self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False) for tx_id in segwit_tx_list: tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash)) - assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash)) - assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"]) - assert(self.nodes[0].getrawtransaction(tx_id, False, blockhash) == bytes_to_hex_str(tx.serialize_without_witness())) + assert self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) @@ -198,11 +198,11 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data - assert(tmpl['weightlimit'] == 4000000) - assert(tmpl['sigoplimit'] == 80000) - assert(tmpl['transactions'][0]['txid'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 8) + assert tmpl['sizelimit'] >= 3999577 # actual maximum size is lower due to minimum mandatory non-witness data + assert tmpl['weightlimit'] == 4000000 + assert tmpl['sigoplimit'] == 80000 + assert tmpl['transactions'][0]['txid'] == txid + assert tmpl['transactions'][0]['sigops'] == 8 self.nodes[0].generate(1) # Mine a block to clear the gbt cache @@ -214,8 +214,8 @@ class SegWitTest(BitcoinTestFramework): txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) hex_tx = self.nodes[0].gettransaction(txid)['hex'] tx = FromHex(CTransaction(), hex_tx) - assert(tx.wit.is_null()) # This should not be a segwit input - assert(txid1 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() # This should not be a segwit input + assert txid1 in self.nodes[0].getrawmempool() # Now create tx2, which will spend from txid1. tx = CTransaction() @@ -224,7 +224,7 @@ class SegWitTest(BitcoinTestFramework): tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx))['hex'] txid2 = self.nodes[0].sendrawtransaction(tx2_hex) tx = FromHex(CTransaction(), tx2_hex) - assert(not tx.wit.is_null()) + assert not tx.wit.is_null() # Now create tx3, which will spend from txid2 tx = CTransaction() @@ -232,15 +232,15 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(int(49.95 * COIN), CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) # Huge fee tx.calc_sha256() txid3 = self.nodes[0].sendrawtransaction(ToHex(tx)) - assert(tx.wit.is_null()) - assert(txid3 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() + assert txid3 in self.nodes[0].getrawmempool() # Check that getblocktemplate includes all transactions. template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) template_txids = [t['txid'] for t in template['transactions']] - assert(txid1 in template_txids) - assert(txid2 in template_txids) - assert(txid3 in template_txids) + assert txid1 in template_txids + assert txid2 in template_txids + assert txid3 in template_txids # Check that wtxid is properly reported in mempool entry assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) @@ -392,22 +392,22 @@ class SegWitTest(BitcoinTestFramework): v = self.nodes[0].getaddressinfo(i) if (v['isscript']): bare = hex_str_to_bytes(v['hex']) - importlist.append(bytes_to_hex_str(bare)) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(bare)]))) + importlist.append(bare.hex()) + importlist.append(CScript([OP_0, sha256(bare)]).hex()) else: pubkey = hex_str_to_bytes(v['pubkey']) p2pk = CScript([pubkey, OP_CHECKSIG]) p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) - importlist.append(bytes_to_hex_str(p2pk)) - importlist.append(bytes_to_hex_str(p2pkh)) - importlist.append(bytes_to_hex_str(CScript([OP_0, hash160(pubkey)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pk)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pkh)]))) + importlist.append(p2pk.hex()) + importlist.append(p2pkh.hex()) + importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) - importlist.append(bytes_to_hex_str(unsolvablep2pkh)) - importlist.append(bytes_to_hex_str(unsolvablep2wshp2pkh)) - importlist.append(bytes_to_hex_str(op1)) - importlist.append(bytes_to_hex_str(p2wshop1)) + importlist.append(unsolvablep2pkh.hex()) + importlist.append(unsolvablep2wshp2pkh.hex()) + importlist.append(op1.hex()) + importlist.append(p2wshop1.hex()) for i in importlist: # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC @@ -535,8 +535,8 @@ class SegWitTest(BitcoinTestFramework): for i in script_list: tx.vout.append(CTxOut(10000000, i)) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - txid = self.nodes[0].sendrawtransaction(signresults, True) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + txid = self.nodes[0].sendrawtransaction(signresults, 0) txs_mined[txid] = self.nodes[0].generate(1)[0] sync_blocks(self.nodes) watchcount = 0 @@ -587,8 +587,8 @@ class SegWitTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(int('0x' + i, 0), j))) tx.vout.append(CTxOut(0, CScript())) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - self.nodes[0].sendrawtransaction(signresults, True) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + self.nodes[0].sendrawtransaction(signresults, 0) self.nodes[0].generate(1) sync_blocks(self.nodes) diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 88df61cabc..0713925141 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -22,7 +22,6 @@ VB_TOP_BITS = 0x20000000 VB_UNKNOWN_BIT = 27 # Choose a bit unassigned to any deployment VB_UNKNOWN_VERSION = VB_TOP_BITS | (1 << VB_UNKNOWN_BIT) -WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible unknown rules are in effect" WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("Warning: unknown new rules activated.*versionbit") @@ -75,18 +74,13 @@ class VersionBitsWarningTest(BitcoinTestFramework): node.generatetoaddress(VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() - assert(not VB_PATTERN.match(node.getmininginfo()["warnings"])) - assert(not VB_PATTERN.match(node.getnetworkinfo()["warnings"])) + assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) + assert not VB_PATTERN.match(node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if >50 blocks in the last 100 were an unknown version") # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(node.p2p, VB_THRESHOLD, VB_UNKNOWN_VERSION) node.generatetoaddress(VB_PERIOD - VB_THRESHOLD, node_deterministic_address) - # Check that get*info() shows the 51/100 unknown block version error. - assert(WARN_UNKNOWN_RULES_MINED in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_MINED in node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if previous VB_BLOCKS have >=VB_THRESHOLD blocks with unknown versionbits version.") # Mine a period worth of expected blocks so the generic block-version warning # is cleared. This will move the versionbit state to ACTIVE. @@ -101,8 +95,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"]) + assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] + assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index aed439339f..15b7ba9ae1 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" @@ -16,7 +16,7 @@ class TestBitcoinCli(BitcoinTestFramework): """Main test logic""" cli_response = self.nodes[0].cli("-version").send_cli() - assert("Bitcoin Core RPC client version" in cli_response) + assert "Bitcoin Core RPC client version" in cli_response self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") if self.is_wallet_compiled(): @@ -57,16 +57,14 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) - if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) - assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) - assert_equal(cli_get_info['testnet'], blockchain_info['chain'] == "test") + assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): + assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index 20889366e5..bb868d7115 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" @@ -30,14 +30,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -47,14 +47,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -64,8 +64,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is None) #now the connection must be closed after the response + assert b'"error":null' in out1 + assert conn.sock is None #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -76,7 +76,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) + assert b'"error":null' in out1 #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urllib.parse.urlparse(self.nodes[2].url) @@ -87,8 +87,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default + assert b'"error":null' in out1 + assert conn.sock is not None #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index f33040242c..a036dfc790 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" @@ -152,7 +152,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) chain_height, = unpack("i", output.read(4)) - response_hash = binascii.hexlify(output.read(32)[::-1]).decode('ascii') + response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine assert_equal(chain_height, 102) # chain height must be 102 @@ -252,7 +252,7 @@ class RESTTest (BitcoinTestFramework): resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) - blockhash = binascii.hexlify(resp_bytes[::-1]).decode('utf-8') + blockhash = resp_bytes[::-1].hex() assert_equal(blockhash, bb_hash) # Check invalid blockhashbyheight requests diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b6955d4492..49ae0fb1a9 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,12 +1,22 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than_or_equal +def expect_http_status(expected_http_status, expected_rpc_code, + fcn, *args): + try: + fcn(*args) + raise AssertionError("Expected RPC error %d, got none" % expected_rpc_code) + except JSONRPCException as exc: + assert_equal(exc.error["code"], expected_rpc_code) + assert_equal(exc.http_status, expected_http_status) + class RPCInterfaceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,9 +58,16 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_equal(result_by_id[3]['error'], None) assert result_by_id[3]['result'] is not None + def test_http_status_codes(self): + self.log.info("Testing HTTP status codes for JSON-RPC requests...") + + expect_http_status(404, -32601, self.nodes[0].invalidmethod) + expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + def run_test(self): self.test_getrpcinfo() self.test_batch_request() + self.test_http_status_codes() if __name__ == '__main__': diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 94fea37090..8e58c85c15 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" @@ -10,7 +10,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction from test_framework.util import ( assert_equal, - bytes_to_hex_str, hash256, ) from io import BytesIO @@ -94,17 +93,17 @@ class ZMQTest (BitcoinTestFramework): tx = CTransaction() tx.deserialize(BytesIO(hex)) tx.calc_sha256() - assert_equal(tx.hash, bytes_to_hex_str(txid)) + assert_equal(tx.hash, txid.hex()) # Should receive the generated block hash. - hash = bytes_to_hex_str(self.hashblock.receive()) + hash = self.hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. - assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) # Should receive the generated raw block. block = self.rawblock.receive() - assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) + assert_equal(genhashes[x], hash256(block[:80]).hex()) if self.is_wallet_compiled(): self.log.info("Wait for tx from second node") @@ -113,11 +112,11 @@ class ZMQTest (BitcoinTestFramework): # Should receive the broadcasted txid. txid = self.hashtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(txid)) + assert_equal(payment_txid, txid.hex()) # Should receive the broadcasted raw transaction. hex = self.rawtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) + assert_equal(payment_txid, hash256(hex).hex()) self.log.info("Test the getzmqnotifications RPC") diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index e2a219b85a..2bb5d8ab7d 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" @@ -27,9 +27,7 @@ from test_framework.script import ( from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str, hex_str_to_bytes, - wait_until, ) @@ -38,7 +36,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ '-txindex', - '-reindex', # Need reindex for txindex '-acceptnonstdtxn=0', # Try to mimic main-net ]] * self.num_nodes @@ -56,7 +53,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 - wait_until(lambda: node.getblockcount() == 200) + assert_equal(node.getblockcount(), 200) assert_equal(node.getmempoolinfo()['size'], self.mempool_size) coins = node.listunspent() @@ -71,7 +68,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): inputs=[{'txid': coin['txid'], 'vout': coin['vout']}], outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], ))['hex'] - txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True) + txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, maxfeerate=0) node.generate(1) self.mempool_size = 0 self.check_mempool_result( @@ -103,10 +100,10 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) - node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True) + node.sendrawtransaction(hexstring=raw_tx_final, maxfeerate=0) self.mempool_size += 1 self.log.info('A transaction in the mempool') @@ -121,7 +118,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(fee * COIN) # Double the fee tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER + 1 # Now, opt out of RBF - raw_tx_0 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] + raw_tx_0 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( @@ -131,15 +128,15 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that conflicts with an unconfirmed tx') # Send the transaction that replaces the mempool transaction and opts out of replaceability - node.sendrawtransaction(hexstring=bytes_to_hex_str(tx.serialize()), allowhighfees=True) + node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) # take original raw_tx_0 tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '18: txn-mempool-conflict'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) self.log.info('A transaction with missing inputs, that never existed') @@ -148,14 +145,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'missing-inputs'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with missing inputs, that existed once in the past') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vin[0].prevout.n = 1 # Set vout to 1, to spend the other outpoint (49 coins) of the in-chain-tx we want to double spend - raw_tx_1 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] - txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, allowhighfees=True) + raw_tx_1 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] + txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, maxfeerate=0) # Now spend both to "clearly hide" the outputs, ie. remove the coins from the utxo set by spending them raw_tx_spend_both = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[ @@ -164,7 +161,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ], outputs=[{node.getnewaddress(): 0.1}] ))['hex'] - txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, allowhighfees=True) + txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, maxfeerate=0) node.generate(1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again @@ -186,17 +183,17 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Reference tx should be valid on itself self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with no outputs') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout = [] # Skip re-signing the transaction for context independent checks from now on - # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex']))) + # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-empty'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A really large transaction') @@ -204,7 +201,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with negative output value') @@ -212,7 +209,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue *= -1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-negative'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large output value') @@ -220,7 +217,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large sum of output values') @@ -229,7 +226,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-txouttotal-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with duplicate inputs') @@ -237,7 +234,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-inputs-duplicate'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A coinbase transaction') @@ -246,7 +243,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_coinbase_spent))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: coinbase'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('Some nonstandard transactions') @@ -254,19 +251,19 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nVersion = 3 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: version'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_0]) # Some non-standard script self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptpubkey'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptsig-not-pushonly'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) @@ -274,21 +271,21 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: tx-size'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0] = output_p2sh_burn tx.vout[0].nValue -= 1 # Make output smaller, such that it is dust for our policy self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: dust'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'\xff']) tx.vout = [tx.vout[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: multi-op-return'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A timelocked transaction') @@ -297,7 +294,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nLockTime = node.getblockcount() + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction that is locked by BIP68 sequence logic') @@ -306,8 +303,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Can skip re-signing the tx because of early rejection self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-BIP68-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index c0918893cd..351b27e94a 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" @@ -47,9 +47,9 @@ class MempoolLimitTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) self.log.info('The tx should be evicted by now') - assert(txid not in self.nodes[0].getrawmempool()) + assert txid not in self.nodes[0].getrawmempool() txdata = self.nodes[0].gettransaction(txid) - assert(txdata['confirmations'] == 0) #confirmation should still be 0 + assert txdata['confirmations'] == 0 #confirmation should still be 0 self.log.info('Check that mempoolminfee is larger than minrelytxfee') assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 9336547a6b..507cdfa9f5 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" @@ -33,7 +33,7 @@ class MempoolPackagesTest(BitcoinTestFramework): signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx['hex']) fulltx = node.getrawtransaction(txid, 1) - assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output + assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output return (txid, send_value) def run_test(self): @@ -58,9 +58,9 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(mempool), MAX_ANCESTORS) descendant_count = 1 descendant_fees = 0 - descendant_size = 0 + descendant_vsize = 0 - ancestor_size = sum([mempool[tx]['size'] for tx in mempool]) + ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool]) ancestor_count = MAX_ANCESTORS ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool]) @@ -79,15 +79,15 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_size += mempool[x]['size'] - assert_equal(mempool[x]['descendantsize'], descendant_size) + descendant_vsize += mempool[x]['vsize'] + assert_equal(mempool[x]['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct assert_equal(mempool[x]['ancestorcount'], ancestor_count) assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_size) - ancestor_size -= mempool[x]['size'] + assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) + ancestor_vsize -= mempool[x]['vsize'] ancestor_fees -= mempool[x]['fee'] ancestor_count -= 1 @@ -125,13 +125,13 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(v_ancestors), len(chain)-1) for x in v_ancestors.keys(): assert_equal(mempool[x], v_ancestors[x]) - assert(chain[-1] not in v_ancestors.keys()) + assert chain[-1] not in v_ancestors.keys() v_descendants = self.nodes[0].getmempooldescendants(chain[0], True) assert_equal(len(v_descendants), len(chain)-1) for x in v_descendants.keys(): assert_equal(mempool[x], v_descendants[x]) - assert(chain[0] not in v_descendants.keys()) + assert chain[0] not in v_descendants.keys() # Check that ancestor modified fees includes fee deltas from # prioritisetransaction diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 845beb551e..187c9026f6 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" @@ -45,7 +45,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 # Use invalidateblock to re-org back for node in self.nodes: @@ -55,7 +55,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] == 0) + assert tx["confirmations"] == 0 # Generate another block, they should all get mined self.nodes[0].generate(1) @@ -63,7 +63,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 if __name__ == '__main__': diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index df8fe23a2e..788aabc192 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -27,7 +27,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str as b2x, connect_nodes_bi, ) from test_framework.script import CScriptNum @@ -36,7 +35,7 @@ from test_framework.script import CScriptNum def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() - rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']}) + rsp = node.getblocktemplate(template_request={'data': block.serialize().hex(), 'mode': 'proposal', 'rules': ['segwit']}) assert_equal(rsp, expect) @@ -64,8 +63,8 @@ class MiningTest(BitcoinTestFramework): def assert_submitblock(block, result_str_1, result_str_2=None): block.solve() result_str_2 = result_str_2 or 'duplicate-invalid' - assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize()))) - assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize()))) + assert_equal(result_str_1, node.submitblock(hexdata=block.serialize().hex())) + assert_equal(result_str_2, node.submitblock(hexdata=block.serialize().hex())) self.log.info('getmininginfo') mining_info = node.getmininginfo() @@ -112,7 +111,7 @@ class MiningTest(BitcoinTestFramework): assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, block.serialize()[:-15].hex()) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -121,10 +120,10 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': block.serialize()[:-1].hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -153,7 +152,7 @@ class MiningTest(BitcoinTestFramework): bad_block_sn = bytearray(block.serialize()) assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1) bad_block_sn[BLOCK_HEADER_SIZE] += 1 - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': bad_block_sn.hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) @@ -193,23 +192,23 @@ class MiningTest(BitcoinTestFramework): return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status} assert chain_tip(block.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(block.serialize())) + node.submitheader(hexdata=block.serialize().hex()) assert chain_tip(block.hash) in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) # Noop + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) # Noop assert chain_tip(block.hash) in node.getchaintips() bad_block_root = copy.deepcopy(block) bad_block_root.hashMerkleRoot += 2 bad_block_root.solve() assert chain_tip(bad_block_root.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() # Should still reject invalid blocks, even if we have the header: - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') assert chain_tip(bad_block_root.hash) in node.getchaintips() # We know the header for this invalid block, so should just return early without error: - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() bad_block_lock = copy.deepcopy(block) @@ -217,19 +216,19 @@ class MiningTest(BitcoinTestFramework): bad_block_lock.vtx[0].rehash() bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root() bad_block_lock.solve() - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal') - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'bad-txns-nonfinal') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'duplicate-invalid') # Build a "good" block on top of the submitted bad block bad_block2 = copy.deepcopy(block) bad_block2.hashPrevBlock = bad_block_lock.sha256 bad_block2.solve() - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) # Should reject invalid header right away bad_block_time = copy.deepcopy(block) bad_block_time.nTime = 1 bad_block_time.solve() - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) # Should ask for the block from a p2p node, if they announce the header as well: node.add_p2p_connection(P2PDataStore()) @@ -240,11 +239,11 @@ class MiningTest(BitcoinTestFramework): # Building a few blocks should give the same results node.generatetoaddress(10, node.get_deterministic_priv_key().address) - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) - assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate') # valid + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) + assert_equal(node.submitblock(hexdata=block.serialize().hex()), 'duplicate') # valid if __name__ == '__main__': diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 72cde8e811..445ec124ce 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" @@ -38,27 +38,27 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): longpollid = template['longpollid'] # longpollid should not change between successive invocations if nothing else happens template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template2['longpollid'] == longpollid) + assert template2['longpollid'] == longpollid # Test 1: test that the longpolling wait if we do nothing thr = LongpollThread(self.nodes[0]) thr.start() # check that thread still lives thr.join(5) # wait 5 seconds or until thread exits - assert(thr.is_alive()) + assert thr.is_alive() # Test 2: test that longpoll will terminate if another node generates a block self.nodes[1].generate(1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 3: test that longpoll will terminate if we generate a block ourselves thr = LongpollThread(self.nodes[0]) thr.start() self.nodes[0].generate(1) # generate a block on another node thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll thr = LongpollThread(self.nodes[0]) @@ -69,7 +69,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) - assert(not thr.is_alive()) + assert not thr.is_alive() if __name__ == '__main__': GetBlockTemplateLPTest().main() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index ca4b621a78..b0a069be81 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -63,9 +63,9 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes = [0, 0, 0] for i in range(3): for j in txids[i]: - assert(j in mempool) - sizes[i] += mempool[j]['size'] - assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count + assert j in mempool + sizes[i] += mempool[j]['vsize'] + assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined @@ -75,8 +75,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that prioritised transaction was mined") - assert(txids[0][0] not in mempool) - assert(txids[0][1] in mempool) + assert txids[0][0] not in mempool + assert txids[0][1] in mempool high_fee_tx = None for x in txids[2]: @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx is not None) + assert high_fee_tx is not None # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). @@ -95,7 +95,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Check to make sure our high fee rate tx is back in the mempool mempool = self.nodes[0].getrawmempool() - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool # Now verify the modified-high feerate transaction isn't mined before # the other high fee transactions. Keep mining until our mempool has @@ -107,14 +107,14 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # transactions should have been. mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool for x in txids[2]: if (x != high_fee_tx): - assert(x not in mempool) + assert x not in mempool # Create a free transaction. Should be rejected. utxo_list = self.nodes[0].listunspent() - assert(len(utxo_list) > 0) + assert len(utxo_list) > 0 utxo = utxo_list[0] inputs = [] @@ -127,7 +127,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # This will raise an exception due to min relay fee not being met assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) - assert(tx_id not in self.nodes[0].getrawmempool()) + assert tx_id not in self.nodes[0].getrawmempool() # This is a less than 1000-byte transaction, so just set the fee # to be the minimum for a 1000-byte transaction and check that it is @@ -136,7 +136,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) - assert(tx_id in self.nodes[0].getrawmempool()) + assert tx_id in self.nodes[0].getrawmempool() # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. @@ -147,7 +147,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].setmocktime(mock_time+10) new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template != new_template) + assert template != new_template if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index f0dee59b5c..3ca6bec254 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -7,19 +7,18 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ -from decimal import Decimal import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_bip9_status, satoshi_round, sync_blocks, wait_until +from test_framework.util import assert_equal, get_bip9_status, wait_until # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self): + def __init__(self, cmpct_version): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -27,6 +26,7 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() + self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -94,11 +94,7 @@ class TestP2PConn(P2PInterface): class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] + self.num_nodes = 1 self.utxos = [] def skip_test_if_missing_module(self): @@ -117,11 +113,10 @@ class CompactBlocksTest(BitcoinTestFramework): # Create 10 more anyone-can-spend utxo's for testing. def make_utxos(self): - # Doesn't matter which node we use, just use node0. block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) - self.nodes[0].generate(100) + self.segwit_node.send_and_ping(msg_block(block)) + assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 + self.nodes[0].generatetoaddress(100, self.nodes[0].getnewaddress(address_type="bech32")) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -135,10 +130,10 @@ class CompactBlocksTest(BitcoinTestFramework): block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() block2.solve() - self.test_node.send_and_ping(msg_block(block2)) + self.segwit_node.send_and_ping(msg_block(block2)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return + # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. @@ -149,7 +144,10 @@ class CompactBlocksTest(BitcoinTestFramework): # are made with compact blocks. # If old_node is passed in, request compact blocks with version=preferred-1 # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + def test_sendcmpct(self, test_node, old_node=None): + preferred_version = test_node.cmpct_version + node = self.nodes[0] + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) @@ -167,7 +165,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.clear_block_announcement() block_hash = int(node.generate(1)[0], 16) peer.wait_for_block_announcement(block_hash, timeout=30) - assert(peer.block_announced) + assert peer.block_announced with mininode_lock: assert predicate(peer), ( @@ -251,23 +249,18 @@ class CompactBlocksTest(BitcoinTestFramework): # This index will be too high prefilled_txn = PrefilledTransaction(1, block.vtx[0]) cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): + def test_compactblock_construction(self, test_node, use_witness_address=True): + version = test_node.cmpct_version + node = self.nodes[0] # Generate a bunch of transactions. node.generate(101) num_transactions = 25 address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.getnewaddress(address_type='bech32') - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) - node.generate(1) segwit_tx_generated = False for i in range(num_transactions): @@ -285,7 +278,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.wait_for_block_announcement(tip) # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) + self.request_cb_announcements(test_node) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -303,7 +296,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -319,7 +312,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -330,7 +323,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(header_and_shortids.header.sha256, block_hash) # Make sure the prefilled_txn appears to have included the coinbase - assert(len(header_and_shortids.prefilled_txn) >= 1) + assert len(header_and_shortids.prefilled_txn) >= 1 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) # Check that all prefilled_txn entries match what's in the block. @@ -345,7 +338,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) else: # Shouldn't have received a witness - assert(entry.tx.wit.is_null()) + assert entry.tx.wit.is_null() # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -375,7 +368,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Post-segwit: upgraded nodes would only make this request of cb-version-2, # NODE_WITNESS peers. Unupgraded nodes would still make this request of # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): + def test_compactblock_requests(self, test_node, segwit=True): + version = test_node.cmpct_version + node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: @@ -407,7 +402,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -440,14 +435,16 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we only receive getblocktxn requests for transactions that the # node needs, and that responding to them causes the block to be # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): + def test_getblocktxn_requests(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert("getblocktxn" in peer.last_message) + assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -487,7 +484,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in node.getrawmempool()) + assert block.vtx[1].hash in node.getrawmempool() # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. @@ -508,7 +505,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Clear out last request. with mininode_lock: @@ -519,13 +516,13 @@ class CompactBlocksTest(BitcoinTestFramework): test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert("getblocktxn" not in test_node.last_message) + assert "getblocktxn" not in test_node.last_message # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() + def test_incorrect_blocktxn_response(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -537,7 +534,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:6]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Send compact block comp_block = HeaderAndShortIDs() @@ -545,7 +542,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -569,7 +566,7 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG) + assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block @@ -579,7 +576,9 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) - def test_getblocktxn_handler(self, node, test_node, version): + def test_getblocktxn_handler(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 @@ -606,7 +605,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: # Witnesses should have been stripped - assert(tx.wit.is_null()) + assert tx.wit.is_null() else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) @@ -626,7 +625,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message - def test_compactblocks_not_at_tip(self, node, test_node): + def test_compactblocks_not_at_tip(self, test_node): + node = self.nodes[0] # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] @@ -669,7 +669,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(x["status"], "headers-only") found = True break - assert(found) + assert found # Requesting this block via getblocktxn should silently fail # (to avoid fingerprinting attacks). @@ -681,11 +681,8 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert "blocktxn" not in test_node.last_message - def activate_segwit(self, node): - node.generate(144 * 3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') - - def test_end_to_end_block_relay(self, node, listeners): + def test_end_to_end_block_relay(self, listeners): + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -706,8 +703,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert(len(self.utxos)) + def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + node = self.nodes[0] + assert len(self.utxos) utxo = self.utxos[0] block = self.build_block_with_transactions(node, utxo, 5) @@ -728,22 +726,24 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg) # Check that the tip didn't advance - assert(int(node.getbestblockhash(), 16) is not block.sha256) + assert int(node.getbestblockhash(), 16) is not block.sha256 test_node.sync_with_ping() # Helper for enabling cb announcements # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): + def request_cb_announcements(self, peer): + node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) msg = msg_sendcmpct() - msg.version = version + msg.version = peer.cmpct_version msg.announce = True peer.send_and_ping(msg) - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert(len(self.utxos)) + def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + node = self.nodes[0] + assert len(self.utxos) def announce_cmpct_block(node, peer): utxo = self.utxos.pop(0) @@ -764,7 +764,7 @@ class CompactBlocksTest(BitcoinTestFramework): delivery_peer.sync_with_ping() mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.sha256) @@ -783,7 +783,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.use_witness = True delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert(int(node.getbestblockhash(), 16) != block.sha256) + assert int(node.getbestblockhash(), 16) != block.sha256 msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 @@ -793,126 +793,55 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) # We will need UTXOs to construct transactions in later tests. self.make_utxos() - self.log.info("Running tests, pre-segwit activation:") + assert_equal(get_bip9_status(self.nodes[0], "segwit")["status"], 'active') self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) + self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) - - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + self.test_compactblock_construction(self.old_node) + self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) + self.test_compactblock_requests(self.segwit_node) self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.segwit_node) self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + self.test_getblocktxn_handler(self.segwit_node) + self.test_getblocktxn_handler(self.old_node) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.segwit_node) + self.test_compactblocks_not_at_tip(self.old_node) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.segwit_node) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.segwit_node, self.additional_segwit_node) # Test that if we submitblock to node1, we'll get a compact block # announcement to all peers. # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + self.request_cb_announcements(self.old_node) + self.request_cb_announcements(self.segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + self.test_invalid_tx_in_compactblock(self.segwit_node) + self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index d589519e45..b9863cf215 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" @@ -57,7 +57,7 @@ class FeeFilterTest(BitcoinTestFramework): # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte @@ -65,7 +65,7 @@ class FeeFilterTest(BitcoinTestFramework): # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received @@ -82,13 +82,13 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() if __name__ == '__main__': diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 336d34a81d..06049db54c 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. @@ -117,9 +117,9 @@ class P2PLeakTest(BitcoinTestFramework): wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) # Make sure no unexpected messages came in - assert(no_version_bannode.unexpected_msg == False) - assert(no_version_idlenode.unexpected_msg == False) - assert(no_verack_idlenode.unexpected_msg == False) + assert no_version_bannode.unexpected_msg == False + assert no_version_idlenode.unexpected_msg == False + assert no_verack_idlenode.unexpected_msg == False if __name__ == '__main__': diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8f8e89cf15..73bfdc868c 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" -from binascii import hexlify import math import random import struct @@ -74,7 +73,6 @@ from test_framework.script import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, connect_nodes, disconnect_nodes, get_bip9_status, @@ -317,7 +315,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_message(msg_witness_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed - assert(tx.hash in self.nodes[0].getrawmempool()) + assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) @@ -335,7 +333,7 @@ class SegWitTest(BitcoinTestFramework): # Verify the hash with witness differs from the txid # (otherwise our testing framework must be broken!) tx.rehash() - assert(tx.sha256 != tx.calc_sha256(with_witness=True)) + assert tx.sha256 != tx.calc_sha256(with_witness=True) # Construct a segwit-signaling block that includes the transaction. block = self.build_next_block(version=(VB_TOP_BITS | (1 << VB_WITNESS_BIT))) @@ -371,20 +369,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) block2 = self.build_next_block(version=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block2, True) block3 = self.build_next_block(version=(VB_TOP_BITS | (1 << 15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -413,8 +411,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, []) # This gives us a witness commitment. - assert(len(block.vtx[0].wit.vtxinwit) == 1) - assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) + assert len(block.vtx[0].wit.vtxinwit) == 1 + assert len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1 test_witness_block(self.nodes[0], self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) @@ -448,7 +446,7 @@ class SegWitTest(BitcoinTestFramework): msg.headers = [CBlockHeader(block4)] self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) - assert(block4.sha256 not in self.old_node.getdataset) + assert block4.sha256 not in self.old_node.getdataset @subtest def test_v0_outputs_arent_spendable(self): @@ -537,7 +535,7 @@ class SegWitTest(BitcoinTestFramework): """Mine enough blocks for segwit's vb state to be 'started'.""" height = self.nodes[0].getblockcount() # Will need to rewrite the tests here if we are past the first period - assert(height < VB_PERIOD - 1) + assert height < VB_PERIOD - 1 # Advance to end of period, status should now be 'started' self.nodes[0].generate(VB_PERIOD - height - 1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') @@ -554,19 +552,19 @@ class SegWitTest(BitcoinTestFramework): # If this is a non-segwit node, we should not get a witness # commitment, nor a version bit signalling segwit. assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) - assert('default_witness_commitment' not in gbt_results) + assert 'default_witness_commitment' not in gbt_results else: # For segwit-aware nodes, check the version bit and the witness # commitment are correct. - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - assert('default_witness_commitment' in gbt_results) + assert block_version & (1 << VB_WITNESS_BIT) != 0 + assert 'default_witness_commitment' in gbt_results witness_commitment = gbt_results['default_witness_commitment'] # Check that default_witness_commitment is present. witness_root = CBlock.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, bytes_to_hex_str(script)) + assert_equal(witness_commitment, script.hex()) @subtest def advance_to_segwit_lockin(self): @@ -575,7 +573,7 @@ class SegWitTest(BitcoinTestFramework): # Advance to end of period, and verify lock-in happens at the end self.nodes[0].generate(VB_PERIOD - 1) height = self.nodes[0].getblockcount() - assert((height % VB_PERIOD) == VB_PERIOD - 2) + assert (height % VB_PERIOD) == VB_PERIOD - 2 assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') @@ -600,7 +598,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_message["getdata"].inv[0].type == 1) + assert self.old_node.last_message["getdata"].inv[0].type == 1 # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -686,13 +684,13 @@ class SegWitTest(BitcoinTestFramework): if self.segwit_status != 'active': # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) @@ -792,7 +790,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # Test the test -- witness serialization should be different - assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) + assert msg_witness_block(block).serialize() != msg_block(block).serialize() # This empty block should be valid. test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -803,7 +801,7 @@ class SegWitTest(BitcoinTestFramework): block_2.solve() # The commitment should have changed! - assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) + assert block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1] # This should also be valid. test_witness_block(self.nodes[0], self.test_node, block_2, accepted=True) @@ -850,7 +848,7 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() block_3.rehash() - assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns + assert len(block_3.vtx[0].vout) == 4 # 3 OP_returns block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=True) @@ -881,19 +879,19 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a' * 5000000) - assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) > MAX_BLOCK_BASE_SIZE # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() != block.hash) + assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() == block.hash) + assert self.nodes[0].getbestblockhash() == block.hash # Now make sure that malleating the witness reserved value doesn't # result in a block permanently marked bad. @@ -918,7 +916,7 @@ class SegWitTest(BitcoinTestFramework): # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() - assert(len(self.utxo) > 0) + assert len(self.utxo) > 0 # Create a P2WSH transaction. # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. @@ -940,7 +938,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_OUTPUTS): parent_tx.vout.append(CTxOut(child_value, script_pubkey)) parent_tx.vout[0].nValue -= 50000 - assert(parent_tx.vout[0].nValue > 0) + assert parent_tx.vout[0].nValue > 0 parent_tx.rehash() child_tx = CTransaction() @@ -970,7 +968,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) # Make sure that our test case would exceed the old max-network-message # limit - assert(len(block.serialize(True)) > 2 * 1024 * 1024) + assert len(block.serialize(True)) > 2 * 1024 * 1024 test_witness_block(self.nodes[0], self.test_node, block, accepted=False) @@ -980,7 +978,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) == MAX_BLOCK_BASE_SIZE test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -998,14 +996,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert(self.nodes[0].getbestblockhash() != block.hash) + self.nodes[0].submitblock(block.serialize(True).hex()) + assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -1020,9 +1018,9 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) + self.nodes[0].submitblock(block_2.serialize(True).hex()) # Tip should not advance! - assert(self.nodes[0].getbestblockhash() != block_2.hash) + assert self.nodes[0].getbestblockhash() != block_2.hash @subtest def test_extra_witness_data(self): @@ -1142,7 +1140,7 @@ class SegWitTest(BitcoinTestFramework): # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) - assert(len(long_witness_program) == MAX_PROGRAM_LENGTH + 1) + assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1166,7 +1164,7 @@ class SegWitTest(BitcoinTestFramework): # Try again with one less byte in the witness program witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) - assert(len(witness_program) == MAX_PROGRAM_LENGTH) + assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) @@ -1197,7 +1195,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(10): tx.vout.append(CTxOut(int(value / 10), script_pubkey)) tx.vout[0].nValue -= 1000 - assert(tx.vout[0].nValue >= 0) + assert tx.vout[0].nValue >= 0 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) @@ -1347,8 +1345,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(raw_tx["vsize"], vsize) assert_equal(raw_tx["weight"], weight) assert_equal(len(raw_tx["vin"][0]["txinwitness"]), 1) - assert_equal(raw_tx["vin"][0]["txinwitness"][0], hexlify(witness_program).decode('ascii')) - assert(vsize != raw_tx["size"]) + assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_program.hex()) + assert vsize != raw_tx["size"] # Cleanup: mine the transactions and update utxo for next test self.nodes[0].generate(1) @@ -1398,7 +1396,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) # Mine all the transactions sync_blocks(self.nodes) - assert(len(self.nodes[0].getrawmempool()) == 0) + assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 1 transactions # are non-standard @@ -1667,7 +1665,7 @@ class SegWitTest(BitcoinTestFramework): # Create a slight bias for producing more utxos num_outputs = random.randint(1, 11) random.shuffle(temp_utxos) - assert(len(temp_utxos) > num_inputs) + assert len(temp_utxos) > num_inputs tx = CTransaction() total_value = 0 for i in range(num_inputs): @@ -1928,7 +1926,7 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Make sure that this peer thinks segwit has activated. - assert(get_bip9_status(self.nodes[2], 'segwit')['status'] == "active") + assert get_bip9_status(self.nodes[2], 'segwit')['status'] == "active" # Make sure this peer's blocks match those of node0. height = self.nodes[2].getblockcount() @@ -1955,7 +1953,7 @@ class SegWitTest(BitcoinTestFramework): extra_sigops_available = MAX_SIGOP_COST % sigops_per_script # We chose the number of checkmultisigs/checksigs to make this work: - assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT + assert extra_sigops_available < 100 # steer clear of MAX_OPS_PER_SCRIPT # This script, when spent with the first # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 11299cbc00..027852809b 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. @@ -114,7 +114,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. @@ -131,7 +131,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h2f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found # But this block should be accepted by node since it has equal work. self.nodes[0].getblock(block_h2f.hash) @@ -150,7 +150,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found self.nodes[0].getblock(block_h3.hash) # But this block should be accepted by node since it has more work. @@ -263,7 +263,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_292.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 3938ca98dd..acc7cd811c 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 31e60f1cea..facb05b54c 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. @@ -35,6 +35,7 @@ from test_framework.util import ( from test_framework.blocktools import ( create_block, create_coinbase, + TIME_GENESIS_BLOCK, ) from test_framework.messages import ( msg_block, @@ -46,9 +47,11 @@ from test_framework.mininode import ( class BlockchainTest(BitcoinTestFramework): def set_test_params(self): + self.setup_clean_chain = True self.num_nodes = 1 def run_test(self): + self.mine_chain() self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) # Set extra args with pruning after rescan is complete self._test_getblockchaininfo() @@ -61,6 +64,15 @@ class BlockchainTest(BitcoinTestFramework): self._test_waitforblockheight() assert self.nodes[0].verifychain(4, 0) + def mine_chain(self): + self.log.info('Create some old blocks') + address = self.nodes[0].get_deterministic_priv_key().address + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + # ten-minute steps from genesis block time + self.nodes[0].setmocktime(t) + self.nodes[0].generatetoaddress(1, address) + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -160,9 +172,9 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(chaintxstats['txcount'], 2) assert_equal(chaintxstats['window_final_block_hash'], b1_hash) assert_equal(chaintxstats['window_block_count'], 0) - assert('window_tx_count' not in chaintxstats) - assert('window_interval' not in chaintxstats) - assert('txrate' not in chaintxstats) + assert 'window_tx_count' not in chaintxstats + assert 'window_interval' not in chaintxstats + assert 'txrate' not in chaintxstats def _test_gettxoutsetinfo(self): node = self.nodes[0] diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 3cc35a7b9a..17dbf59a8c 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -90,7 +90,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) self.moved += outval - tx = node0.sendrawtransaction(rawtx3["hex"], True) + tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = node0.generate(1)[0] assert tx in node0.getblock(blk)["tx"] diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 940386eee7..01b8cb1854 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" from test_framework.messages import CTransaction, sha256 from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, hex_str_to_bytes from io import BytesIO @@ -81,7 +81,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(multisig_script) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH - multisig_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(multisig_script))) + multisig_script_hash = sha256(hex_str_to_bytes(multisig_script)).hex() assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) # 4) P2SH scriptPubKey @@ -119,7 +119,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(cltv_script) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH - cltv_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(cltv_script))) + cltv_script_hash = sha256(hex_str_to_bytes(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) # 7) P2PK scriptPubKey @@ -196,7 +196,7 @@ class DecodeScriptTest(BitcoinTestFramework): # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). - push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] + push_signature = txSave.vin[0].scriptSig.hex()[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' @@ -206,23 +206,23 @@ class DecodeScriptTest(BitcoinTestFramework): # 1) P2PK scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) def run_test(self): diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 588bfbe083..9a21998d11 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,32 +1,29 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error +# from test_framework.util import assert_raises_rpc_error class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], ["-deprecatedrpc=generate"]] - - def skip_test_if_missing_module(self): - # The generate RPC method requires the wallet to be compiled - self.skip_if_no_wallet() + self.extra_args = [[], []] def run_test(self): # This test should be used to verify correct behaviour of deprecated # RPC methods with and without the -deprecatedrpc flags. For example: # - # self.log.info("Make sure that -deprecatedrpc=createmultisig allows it to take addresses") - # assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, [self.nodes[0].getnewaddress()]) - # self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) - - self.log.info("Test generate RPC") - assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) - self.nodes[1].generate(1) + # In set_test_params: + # self.extra_args = [[], ["-deprecatedrpc=generate"]] + # + # In run_test: + # self.log.info("Test generate RPC") + # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) + # self.nodes[1].generate(1) + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py index c8b58cfc74..1984694692 100755 --- a/test/functional/rpc_deriveaddresses.py +++ b/test/functional/rpc_deriveaddresses.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the deriveaddresses rpc call.""" @@ -27,17 +27,20 @@ class DeriveaddressesTest(BitcoinTestFramework): assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)#kft60nuy" - assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 0, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, [1, 2]), ["bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) - assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), 0, 2) + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), [0, 2]) assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)")) - assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 0) + assert_raises_rpc_error(-8, "End of range is too high", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 10000000000) - assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 2, 0) + assert_raises_rpc_error(-8, "Range is too large", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [1000000000, 2000000000]) - assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), -1, 0) + assert_raises_rpc_error(-8, "Range specified as [begin,end] must not have begin after end", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [2, 0]) + + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [-1, 0]) combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)") assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 4f350953b2..d89fd6461f 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -94,7 +94,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test that we have enough inputs + assert len(dec_tx['vin']) > 0 #test that we have enough inputs ############################## # simple test with two coins # @@ -107,7 +107,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test if we have enough inputs + assert len(dec_tx['vin']) > 0 #test if we have enough inputs ############################## # simple test with two coins # @@ -120,7 +120,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -139,7 +139,7 @@ class RawTransactionsTest(BitcoinTestFramework): for out in dec_tx['vout']: totalOut += out['value'] - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -363,7 +363,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ ############################################################ @@ -378,7 +378,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -405,7 +405,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -438,7 +438,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -558,7 +558,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance*19) #~19 inputs + assert feeDelta >= 0 and feeDelta <= feeTolerance*19 #~19 inputs ############################################# @@ -620,7 +620,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(res_dec["vin"]), 1) assert_equal(res_dec["vin"][0]["txid"], watchonly_txid) - assert("fee" in result.keys()) + assert "fee" in result.keys() assert_greater_than(result["changepos"], -1) ############################################################### @@ -635,16 +635,16 @@ class RawTransactionsTest(BitcoinTestFramework): result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) - assert(res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid) + assert res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid assert_greater_than(result["fee"], 0) assert_greater_than(result["changepos"], -1) assert_equal(result["fee"] + res_dec["vout"][result["changepos"]]["value"], watchonly_amount / 10) signedtx = self.nodes[3].signrawtransactionwithwallet(result["hex"]) - assert(not signedtx["complete"]) + assert not signedtx["complete"] signedtx = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) - assert(signedtx["complete"]) + assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) self.nodes[0].generate(1) self.sync_all() @@ -676,10 +676,10 @@ class RawTransactionsTest(BitcoinTestFramework): for out in res_dec['vout']: if out['value'] > 1.0: changeaddress += out['scriptPubKey']['addresses'][0] - assert(changeaddress != "") + assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() # Now the change address key should be removed from the keypool - assert(changeaddress != nextaddr) + assert changeaddress != nextaddr ###################################### # Test subtractFeeFromOutputs option # diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index ca9e24367a..e17a8f6421 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -178,5 +178,10 @@ class GetblockstatsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats, hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f') + # Invalid number of args + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats, '00', 1, 2) + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats) + + if __name__ == '__main__': GetblockstatsTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py new file mode 100755 index 0000000000..7bf8e68176 --- /dev/null +++ b/test/functional/rpc_misc.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPC misc output.""" +import xml.etree.ElementTree as ET + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, +) + +from test_framework.authproxy import JSONRPCException + + +class RpcMiscTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + node = self.nodes[0] + + self.log.info("test getmemoryinfo") + memory = node.getmemoryinfo()['locked'] + assert_greater_than(memory['used'], 0) + assert_greater_than(memory['free'], 0) + assert_greater_than(memory['total'], 0) + # assert_greater_than_or_equal() for locked in case locking pages failed at some point + assert_greater_than_or_equal(memory['locked'], 0) + assert_greater_than(memory['chunks_used'], 0) + assert_greater_than(memory['chunks_free'], 0) + assert_equal(memory['used'] + memory['free'], memory['total']) + + self.log.info("test mallocinfo") + try: + mallocinfo = node.getmemoryinfo(mode="mallocinfo") + self.log.info('getmemoryinfo(mode="mallocinfo") call succeeded') + tree = ET.fromstring(mallocinfo) + assert_equal(tree.tag, 'malloc') + except JSONRPCException: + self.log.info('getmemoryinfo(mode="mallocinfo") not available') + assert_raises_rpc_error(-8, 'mallocinfo is only available when compiled with glibc 2.10+', node.getmemoryinfo, mode="mallocinfo") + + assert_raises_rpc_error(-8, "unknown mode foobar", node.getmemoryinfo, mode="foobar") + +if __name__ == '__main__': + RpcMiscTest().main() diff --git a/test/functional/rpc_named_arguments.py b/test/functional/rpc_named_arguments.py index 6c5c29dfdf..ecac9c2f82 100755 --- a/test/functional/rpc_named_arguments.py +++ b/test/functional/rpc_named_arguments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test using named arguments for RPCs.""" @@ -17,7 +17,7 @@ class NamedArgumentTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] h = node.help(command='getblockchaininfo') - assert(h.startswith('getblockchaininfo\n')) + assert h.startswith('getblockchaininfo\n') assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getblockchaininfo') diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 72e6e6329f..da228df9c3 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" @@ -16,7 +16,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blockhash = node_src.getbestblockhash() while True: try: - assert(len(node_dest.getblock(blockhash, False)) > 0) + assert len(node_dest.getblock(blockhash, False)) > 0 break except: blocks_to_copy.append(blockhash) @@ -24,7 +24,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blocks_to_copy.reverse() for blockhash in blocks_to_copy: blockdata = node_src.getblock(blockhash, False) - assert(node_dest.submitblock(blockdata) in (None, 'inconclusive')) + assert node_dest.submitblock(blockdata) in (None, 'inconclusive') def node_sync_via_rpc(nodes): for node_src in nodes: @@ -57,7 +57,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine competing blocks E-F-G on Node 1") hashG = self.nodes[1].generatetoaddress(3, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 5) - assert(hashC != hashG) + assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index e9098e4f5a..904ddac428 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. @@ -47,11 +47,11 @@ class PSBTTest(BitcoinTestFramework): utxos = online_node.listunspent(addresses=[offline_addr]) raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] - assert("non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]) + assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] - assert("witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0]) + assert "witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) @@ -355,7 +355,7 @@ class PSBTTest(BitcoinTestFramework): assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program'] # Check fee and size things - assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == '0.00746268 BTC/kB' + assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == Decimal('0.00746268') # After signing and finalizing, needs extracting signed = self.nodes[1].walletprocesspsbt(updated)['psbt'] diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index c793cf00de..8c82b0ae4d 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -17,7 +17,7 @@ from decimal import Decimal from io import BytesIO from test_framework.messages import CTransaction, ToHex from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes class multidict(dict): """Dictionary that allows duplicate keys. @@ -119,21 +119,21 @@ class RawTransactionsTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs={address: 99})))) assert_equal(len(tx.vout), 1) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))))) assert_equal(len(tx.vout), 2) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) # Multiple mixed outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))))) assert_equal(len(tx.vout), 3) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) @@ -285,11 +285,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "amount" : vout['value']}] @@ -330,11 +326,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx2['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx2['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex'], "amount" : vout['value']}] @@ -434,5 +426,30 @@ class RawTransactionsTest(BitcoinTestFramework): decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + self.log.info('sendrawtransaction/testmempoolaccept with maxfeerate') + + txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) + rawTx = self.nodes[0].getrawtransaction(txId, True) + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) + + self.sync_all() + inputs = [{ "txid" : txId, "vout" : vout['n'] }] + outputs = { self.nodes[0].getnewaddress() : Decimal("0.99999000") } # 1000 sat fee + rawTx = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) + assert_equal(rawTxSigned['complete'], True) + # 1000 sat fee, ~200 b transaction, fee rate should land around 5 sat/b = 0.00005000 BTC/kB + # Thus, testmempoolaccept should reject + testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] + assert_equal(testres['allowed'], False) + assert_equal(testres['reject-reason'], '256: absurdly-high-fee') + # and sendrawtransaction should throw + assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + # And below calls should both succeed + testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate=0.00007000)[0] + assert_equal(testres['allowed'], True) + self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate=0.00007000) + + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 29074c6af3..6346477922 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" @@ -95,6 +95,7 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1499}])['total_amount'], Decimal("12.288")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) + assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": [1500,1500]}])['total_amount'], Decimal("16.384")) # Test the reported descriptors for a few matches assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"]) diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessage.py index ad0e29b451..0cb3ce4215 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing and verifying messages.""" @@ -25,18 +25,18 @@ class SignMessagesTest(BitcoinTestFramework): expected_signature = 'INbVnW4e6PeRmsv2Qgu8NuopvrVjkcxob+sX8OcZG0SALhWybUjzMLPdAsXI46YZGb0KQTRii+wWIQzRpG/U+S0=' signature = self.nodes[0].signmessagewithprivkey(priv_key, message) assert_equal(expected_signature, signature) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test signing with an address with wallet') address = self.nodes[0].getnewaddress() signature = self.nodes[0].signmessage(address, message) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test verifying with another address should not work') other_address = self.nodes[0].getnewaddress() other_signature = self.nodes[0].signmessage(other_address, message) - assert(not self.nodes[0].verifymessage(other_address, signature, message)) - assert(not self.nodes[0].verifymessage(address, other_signature, message)) + assert not self.nodes[0].verifymessage(other_address, signature, message) + assert not self.nodes[0].verifymessage(address, other_signature, message) if __name__ == '__main__': SignMessagesTest().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 56e2c73a90..780758e219 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes from test_framework.messages import sha256 from test_framework.script import CScript, OP_0 @@ -15,7 +15,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-deprecatedrpc=signrawtransaction"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -161,7 +160,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0] assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"]) p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))]) - assert_equal(unspent_output["redeemScript"], bytes_to_hex_str(p2sh_redeemScript)) + assert_equal(unspent_output["redeemScript"], p2sh_redeemScript.hex()) # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")}) spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 20b6341550..e86f91b1d0 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC call related to the uptime command. @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) - assert(self.nodes[0].uptime() >= wait_time) + assert self.nodes[0].uptime() >= wait_time if __name__ == '__main__': diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 456d43aa2e..f36cffe957 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" from .script import hash256, hash160, sha256, CScript, OP_0 -from .util import bytes_to_hex_str, hex_str_to_bytes +from .util import hex_str_to_bytes from . import segwit_addr @@ -16,9 +16,9 @@ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def byte_to_base58(b, version): result = '' - str = bytes_to_hex_str(b) - str = bytes_to_hex_str(chr(version).encode('latin-1')) + str - checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str))) + str = b.hex() + str = chr(version).encode('latin-1').hex() + str + checksum = hash256(hex_str_to_bytes(str)).hex() str += checksum[:8] value = int('0x'+str,0) while value > 0: @@ -32,12 +32,12 @@ def byte_to_base58(b, version): # TODO: def base58_decode def keyhash_to_p2pkh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 0 if main else 111 return byte_to_base58(hash, version) def scripthash_to_p2sh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 5 if main else 196 return byte_to_base58(hash, version) @@ -80,11 +80,11 @@ def check_key(key): key = hex_str_to_bytes(key) # Assuming this is hex string if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): return key - assert(False) + assert False def check_script(script): if (type(script) is str): script = hex_str_to_bytes(script) # Assuming this is hex string if (type(script) is bytes or type(script) is CScript): return script - assert(False) + assert False diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 1140fe9b3e..d039f7d6fe 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -35,6 +35,7 @@ ServiceProxy class: import base64 import decimal +from http import HTTPStatus import http.client import json import logging @@ -49,13 +50,14 @@ USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): - def __init__(self, rpc_error): + def __init__(self, rpc_error, http_status=None): try: errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' super().__init__(errmsg) self.error = rpc_error + self.http_status = http_status def EncodeDecimal(o): @@ -131,19 +133,26 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - response = self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: - raise JSONRPCException(response['error']) + raise JSONRPCException(response['error'], status) elif 'result' not in response: raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}) + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) else: return response['result'] def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) - return self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + return response def _get_response(self): req_start_time = time.time() @@ -162,8 +171,9 @@ class AuthServiceProxy(): content_type = http_response.getheader('Content-Type') if content_type != 'application/json': - raise JSONRPCException({ - 'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}) + raise JSONRPCException( + {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, + http_response.status) responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) @@ -172,7 +182,7 @@ class AuthServiceProxy(): log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) - return response + return response, http_response.status def __truediv__(self, relative_uri): return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 15f4502994..7ac044d0d0 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -20,7 +20,6 @@ from .messages import ( CTxOut, FromHex, ToHex, - bytes_to_hex_str, hash256, hex_str_to_bytes, ser_string, @@ -132,7 +131,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C Can optionally pass scriptPubKey and scriptSig, default is anyone-can-spend output. """ tx = CTransaction() - assert(n < len(prevtx.vout)) + assert n < len(prevtx.vout) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff)) tx.vout.append(CTxOut(amount, script_pub_key)) tx.calc_sha256() @@ -190,7 +189,7 @@ def witness_script(use_p2wsh, pubkey): witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) scripthash = sha256(witness_program) pkscript = CScript([OP_0, scripthash]) - return bytes_to_hex_str(pkscript) + return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): """Return a transaction (in hex) that spends the given utxo to a segwit output. @@ -215,7 +214,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) if (sign): signed = node.signrawtransactionwithwallet(tx_to_witness) - assert("errors" not in signed or len(["errors"]) == 0) + assert "errors" not in signed or len(["errors"]) == 0 return node.sendrawtransaction(signed["hex"]) else: if (insert_redeem_script): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4bd58519c5..7cf51d9223 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -28,7 +28,7 @@ import struct import time from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, assert_equal +from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -181,7 +181,7 @@ def FromHex(obj, hex_string): # Convert a binary-serializable object to hex (eg for submission via RPC) def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) + return obj.serialize().hex() # Objects that map to bitcoind objects, which can be serialized/deserialized @@ -319,7 +319,7 @@ class CTxIn: def __repr__(self): return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + % (repr(self.prevout), self.scriptSig.hex(), self.nSequence) @@ -343,7 +343,7 @@ class CTxOut: def __repr__(self): return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) + self.scriptPubKey.hex()) class CScriptWitness: @@ -355,7 +355,7 @@ class CScriptWitness: def __repr__(self): return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) + (",".join([x.hex() for x in self.stack])) def is_null(self): if self.stack: diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ac7cc068bd..7a063ac526 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -171,7 +171,7 @@ class P2PConnection(asyncio.Protocol): if len(self.recvbuf) < 4: return if self.recvbuf[:4] != self.magic_bytes: - raise ValueError("got garbage %s" % repr(self.recvbuf)) + raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] @@ -218,10 +218,7 @@ class P2PConnection(asyncio.Protocol): def maybe_write(): if not self._transport: return - # Python <3.4.4 does not have is_closing, so we have to check for - # its existence explicitly as long as Bitcoin Core supports all - # Python 3.4 versions. - if hasattr(self._transport, 'is_closing') and self._transport.is_closing(): + if self._transport.is_closing(): return self._transport.write(raw_message_bytes) NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write) diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 45c182f630..c98424e8e2 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. @@ -12,7 +12,7 @@ import socket import struct import array import os -from binascii import unhexlify, hexlify +from binascii import unhexlify # STATE_ESTABLISHED = '01' # STATE_SYN_SENT = '02' @@ -129,17 +129,17 @@ def addr_to_hex(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr = sub[0] + ([0] * nullbytes) + sub[1] else: raise ValueError('Could not parse address %s' % addr) - return hexlify(bytearray(addr)).decode('ascii') + return bytearray(addr).hex() def test_ipv6_local(): ''' diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 012c80a1be..384062b808 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as SignatureHash(). @@ -9,7 +9,6 @@ This file is modified from python-bitcoinlib. from .messages import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string -from binascii import hexlify import hashlib import struct @@ -450,10 +449,6 @@ class CScript(bytes): # join makes no sense for a CScript() raise NotImplementedError - # Python 3.4 compatibility - def hex(self): - return hexlify(self).decode('ascii') - def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): return super(CScript, cls).__new__(cls, value) @@ -545,7 +540,7 @@ class CScript(bytes): def __repr__(self): def _repr(o): if isinstance(o, bytes): - return "x('%s')" % hexlify(o).decode('ascii') + return "x('%s')" % o.hex() else: return repr(o) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index a21c864e75..799b1c74b8 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" @@ -144,7 +144,7 @@ class Socks5Server(): thread.start() def start(self): - assert(not self.running) + assert not self.running self.running = True self.thread = threading.Thread(None, self.run) self.thread.daemon = True diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 869b36673e..15d2e08c93 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -29,11 +29,11 @@ from .util import ( get_datadir_path, initialize_datadir, p2p_port, - set_node_times, sync_blocks, sync_mempools, ) + class TestStatus(Enum): PASSED = 1 FAILED = 2 @@ -94,7 +94,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.setup_clean_chain = False self.nodes = [] self.network_thread = None - self.mocktime = 0 self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = False self.bind_to_localhost_only = True @@ -275,6 +274,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.add_nodes(self.num_nodes, extra_args) self.start_nodes() self.import_deterministic_coinbase_privkeys() + if not self.setup_clean_chain: + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) + # To ensure that all nodes are out of IBD, the most recent block + # must have a timestamp not too old (see IsInitialBlockDownload()). + self.log.debug('Generate a block with current time') + block_hash = self.nodes[0].generate(1)[0] + block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) + for n in self.nodes: + n.submitblock(block) + chain_info = n.getblockchaininfo() + assert_equal(chain_info["blocks"], 200) + assert_equal(chain_info["initialblockdownload"], False) def import_deterministic_coinbase_privkeys(self): for n in self.nodes: @@ -316,7 +328,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): timewait=self.rpc_timeout, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, - mocktime=self.mocktime, coverage_dir=self.options.coveragedir, cwd=self.options.tmpdir, extra_conf=extra_confs[i], @@ -435,7 +446,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. - Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Create a cache of a 199-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" assert self.num_nodes <= MAX_NODES @@ -468,7 +479,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): timewait=self.rpc_timeout, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, - mocktime=self.mocktime, coverage_dir=None, cwd=self.options.tmpdir, )) @@ -479,32 +489,22 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: node.wait_for_rpc_connection() - # For backward compatibility of the python scripts with previous - # versions of the cache, set mocktime to Jan 1, - # 2014 + (201 * 10 * 60)""" - self.mocktime = 1388534400 + (201 * 10 * 60) - - # Create a 200-block-long chain; each of the 4 first nodes + # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. - # Note: To preserve compatibility with older versions of - # initialize_chain, only 4 nodes will generate coins. - # - # blocks are created with timestamps 10 minutes apart - # starting from 2010 minutes in the past - block_time = self.mocktime - (201 * 10 * 60) - for i in range(2): - for peer in range(4): - for j in range(25): - set_node_times(self.nodes, block_time) - self.nodes[peer].generatetoaddress(1, self.nodes[peer].get_deterministic_priv_key().address) - block_time += 10 * 60 - # Must sync before next peer starts generating blocks - sync_blocks(self.nodes) + # The 4th node gets only 24 immature blocks so that the very last + # block in the cache does not age too much (have an old tip age). + # This is needed so that we are out of IBD when the test starts, + # see the tip age check in IsInitialBlockDownload(). + for i in range(8): + self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address) + sync_blocks(self.nodes) + + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] - self.mocktime = 0 def cache_path(n, *paths): return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 352774914d..ec5d3b267e 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -31,9 +31,6 @@ from .util import ( p2p_port, ) -# For Python 3.4 compatibility -JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) - BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -61,7 +58,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -90,8 +87,7 @@ class TestNode(): "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", - "-mocktime=" + str(mocktime), - "-uacomment=testnode%d" % i + "-uacomment=testnode%d" % i, ] self.cli = TestNodeCLI(bitcoin_cli, self.datadir) @@ -566,5 +562,5 @@ class TestNodeCLI(): raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except JSONDecodeError: + except json.JSONDecodeError: return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index fef9982412..f771a37b99 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" from base64 import b64encode -from binascii import hexlify, unhexlify +from binascii import unhexlify from decimal import Decimal, ROUND_DOWN import hashlib import inspect @@ -182,9 +182,6 @@ def check_json_precision(): def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def bytes_to_hex_str(byte_str): - return hexlify(byte_str).decode('ascii') - def hash256(byte_str): sha256 = hashlib.sha256() sha256.update(byte_str) @@ -267,7 +264,7 @@ def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): - assert(n <= MAX_NODES) + assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): @@ -425,7 +422,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ - assert(confirmations_required >= 0) + assert confirmations_required >= 0 utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] @@ -470,7 +467,7 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransactionwithwallet(rawtx) - txid = from_node.sendrawtransaction(signresult["hex"], True) + txid = from_node.sendrawtransaction(signresult["hex"], 0) return (txid, signresult["hex"], fee) @@ -503,7 +500,7 @@ def create_confirmed_utxos(fee, node, count): node.generate(1) utxos = node.listunspent() - assert(len(utxos) >= count) + assert len(utxos) >= count return utxos # Create large OP_RETURN txouts that can be appended to a transaction @@ -542,7 +539,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): newtx = newtx + txouts newtx = newtx + rawtx[94:] signresult = node.signrawtransactionwithwallet(newtx, None, "NONE") - txid = node.sendrawtransaction(signresult["hex"], True) + txid = node.sendrawtransaction(signresult["hex"], 0) txids.append(txid) return txids diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 999cd72108..61c5c9aefb 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -110,6 +110,7 @@ BASE_SCRIPTS = [ 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', + 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'mempool_reorg.py', @@ -477,6 +478,11 @@ class TestHandler: log_stderr)) if not self.jobs: raise IndexError('pop from empty list') + + # Print remaining running jobs when all jobs have been started. + if not self.test_list: + print("Remaining jobs: [{}]".format(", ".join(j[0] for j in self.jobs))) + dot_count = 0 while True: # Return first proc that finishes diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 380dd9c0ad..a44aa5a18e 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -102,62 +102,62 @@ class AddressTypeTest(BitcoinTestFramework): def test_address(self, node, address, multisig, typ): """Run sanity checks on an address.""" info = self.nodes[node].getaddressinfo(address) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] assert_equal(info.get('solvable'), True) if not multisig and typ == 'legacy': # P2PKH - assert(not info['isscript']) - assert(not info['iswitness']) - assert('pubkey' in info) + assert not info['isscript'] + assert not info['iswitness'] + assert 'pubkey' in info elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert(info['isscript']) - assert(not info['iswitness']) + assert info['isscript'] + assert not info['iswitness'] assert_equal(info['script'], 'witness_v0_keyhash') - assert('pubkey' in info) + assert 'pubkey' in info elif not multisig and typ == 'bech32': # P2WPKH - assert(not info['isscript']) - assert(info['iswitness']) + assert not info['isscript'] + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 40) - assert('pubkey' in info) + assert 'pubkey' in info elif typ == 'legacy': # P2SH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(not info['iswitness']) - assert('pubkeys' in info) + assert not info['iswitness'] + assert 'pubkeys' in info elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'witness_v0_scripthash') - assert(not info['iswitness']) - assert(info['embedded']['isscript']) + assert not info['iswitness'] + assert info['embedded']['isscript'] assert_equal(info['embedded']['script'], 'multisig') - assert(info['embedded']['iswitness']) + assert info['embedded']['iswitness'] assert_equal(info['embedded']['witness_version'], 0) assert_equal(len(info['embedded']['witness_program']), 64) - assert('pubkeys' in info['embedded']) + assert 'pubkeys' in info['embedded'] elif typ == 'bech32': # P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(info['iswitness']) + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 64) - assert('pubkeys' in info) + assert 'pubkeys' in info else: # Unknown type - assert(False) + assert False def test_desc(self, node, address, multisig, typ, utxo): """Run sanity checks on a descriptor reported by getaddressinfo.""" info = self.nodes[node].getaddressinfo(address) - assert('desc' in info) + assert 'desc' in info assert_equal(info['desc'], utxo['desc']) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] # Use a ridiculously roundabout way to find the key origin info through # the PSBT logic. However, this does test consistency between the PSBT reported @@ -172,11 +172,11 @@ class AddressTypeTest(BitcoinTestFramework): key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] # Verify the descriptor checksum against the Python implementation - assert(descsum_check(info['desc'])) + assert descsum_check(info['desc']) # Verify that stripping the checksum and recreating it using Python roundtrips - assert(info['desc'] == descsum_create(info['desc'][:-9])) + assert info['desc'] == descsum_create(info['desc'][:-9]) # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips - assert(info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor']) + assert info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor'] if not multisig and typ == 'legacy': # P2PKH @@ -198,7 +198,7 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) else: # Unknown type - assert(False) + assert False def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 05c97e0340..3a65b684c8 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -129,5 +129,17 @@ class WalletTest(BitcoinTestFramework): # getbalance with minconf=2 will show the new balance. assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0')) + # check mempool transactions count for wallet unconfirmed balance after + # dynamically loading the wallet. + before = self.nodes[1].getunconfirmedbalance() + dst = self.nodes[1].getnewaddress() + self.nodes[1].unloadwallet('') + self.nodes[0].sendtoaddress(dst, 0.1) + self.sync_all() + self.nodes[1].loadwallet('') + after = self.nodes[1].getunconfirmedbalance() + assert_equal(before + Decimal('0.1'), after) + + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index a5df763eb0..c89f42fc8a 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -11,11 +11,9 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_fee_amount, - assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, sync_blocks, - sync_mempools, wait_until, ) @@ -84,13 +82,8 @@ class WalletTest(BitcoinTestFramework): assert_equal(txout['value'], 50) # Send 21 BTC from 0 to 2 using sendtoaddress call. - # Locked memory should increase to sign transactions - self.log.info("test getmemoryinfo") - memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - memory_after = self.nodes[0].getmemoryinfo() - assert_greater_than(memory_after['locked']['used'], memory_before['locked']['used']) self.log.info("test gettxout (second part)") # utxo spent in mempool should be visible if you exclude mempool @@ -174,8 +167,8 @@ class WalletTest(BitcoinTestFramework): txns_to_send.append(self.nodes[0].signrawtransactionwithwallet(raw_tx)) # Have node 1 (miner) send the transactions - self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], True) - self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], True) + self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], 0) + self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], 0) # Have node1 mine a block to confirm transactions: self.nodes[1].generate(1) @@ -222,23 +215,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) - # Test ResendWalletTransactions: - # Create a couple of transactions, then start up a fourth - # node (nodes[3]) and ask nodes[0] to rebroadcast. - # EXPECT: nodes[3] should have those transactions in its mempool. - txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - sync_mempools(self.nodes[0:2]) - self.start_node(3) connect_nodes_bi(self.nodes, 0, 3) - sync_blocks(self.nodes) - - relayed = self.nodes[0].resendwallettransactions() - assert_equal(set(relayed), {txid1, txid2}) - sync_mempools(self.nodes) - - assert txid1 in self.nodes[3].getrawmempool() + self.sync_all() # check if we can list zero value tx as available coins # 1. create raw_tx diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 7d3d9b61e2..fc752e5ac0 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -19,7 +19,7 @@ import io from test_framework.blocktools import add_witness_commitment, create_block, create_coinbase, send_to_witness from test_framework.messages import BIP125_SEQUENCE_NUMBER, CTransaction from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes, sync_mempools +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes, sync_mempools WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -298,7 +298,7 @@ def submit_block_with_tx(node, tx): block.hashMerkleRoot = block.calc_merkle_root() add_witness_commitment(block) block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) return block diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 27dc0fb279..0b584a0bb2 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,17 +7,25 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) +from test_framework.blocktools import ( + TIME_GENESIS_BLOCK, +) class CreateTxWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False + self.setup_clean_chain = True self.num_nodes = 1 def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.log.info('Create some old blocks') + self.nodes[0].setmocktime(TIME_GENESIS_BLOCK) + self.nodes[0].generate(200) + self.nodes[0].setmocktime(0) + self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 7ec72b8649..8ec4b98b6e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet arguments. @@ -40,8 +40,8 @@ class CreateWalletTest(BitcoinTestFramework): privkey = w0.dumpprivkey(addr) assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) - assert(not result[0]['success']) - assert('warning' not in result[0]) + assert not result[0]['success'] + assert 'warning' not in result[0] assert_equal(result[0]['error']['code'], -4) assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 6530c58c78..7c2ec56b5a 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. @@ -21,9 +21,9 @@ class DisableWalletTest (BitcoinTestFramework): # Make sure wallet is really disabled assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') - assert(x['isvalid'] == False) + assert x['isvalid'] == False x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert(x['isvalid'] == True) + assert x['isvalid'] == True # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 3f39654bb8..53edf710b9 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" @@ -46,10 +46,10 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): keypath = None if keytype == "inactivehdseed=1": # ensure the old master is still available - assert (hd_master_addr_old == addr) + assert hd_master_addr_old == addr elif keytype == "hdseed=1": # ensure we have generated a new hd master key - assert (hd_master_addr_old != addr) + assert hd_master_addr_old != addr hd_master_addr_ret = addr elif keytype == "script=1": # scripts don't have keypaths diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 61245e5104..97172d8b82 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 8e25aa7337..5bfbaa2f0b 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC. @@ -25,7 +25,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - bytes_to_hex_str, ) from test_framework.wallet_util import ( get_key, @@ -127,7 +126,7 @@ class ImportMultiTest(BitcoinTestFramework): # Nonstandard scriptPubKey + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal flag") - nonstandardScriptPubKey = key.p2pkh_script + bytes_to_hex_str(CScript([OP_NOP])) + nonstandardScriptPubKey = key.p2pkh_script + CScript([OP_NOP]).hex() key = get_key(self.nodes[0]) self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, "timestamp": "now"}, @@ -584,7 +583,7 @@ class ImportMultiTest(BitcoinTestFramework): self.log.info("Should import the ranged descriptor with specified range as solvable") self.test_importmulti({"desc": descsum_create(desc), "timestamp": "now", - "range": {"end": 1}}, + "range": 1}, success=True, warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) for address in addresses: @@ -807,7 +806,7 @@ class ImportMultiTest(BitcoinTestFramework): 'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), 'keypool': True, 'timestamp': 'now', - 'range' : {'start': 0, 'end': 4} + 'range' : [0, 4], }] ) for i in range(0, 5): diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 1116196268..e3aeb61197 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" @@ -21,7 +21,7 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') @@ -29,8 +29,8 @@ class KeyPoolTest(BitcoinTestFramework): addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid']) - assert(addr_data['hdseedid'] == wallet_info['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -59,7 +59,7 @@ class KeyPoolTest(BitcoinTestFramework): addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) - assert(len(addr) == 6) + assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index b7c8d3098d..e099bab6a3 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. @@ -55,11 +55,11 @@ class KeypoolRestoreTest(BitcoinTestFramework): # Make sure we're creating the outputs we expect address_details = self.nodes[idx].validateaddress(addr_extpool) if i == 0: - assert(not address_details["isscript"] and not address_details["iswitness"]) + assert not address_details["isscript"] and not address_details["iswitness"] elif i == 1: - assert(address_details["isscript"] and not address_details["iswitness"]) + assert address_details["isscript"] and not address_details["iswitness"] else: - assert(not address_details["isscript"] and address_details["iswitness"]) + assert not address_details["isscript"] and address_details["iswitness"] self.log.info("Send funds to wallet") diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 17f044bf65..83f7ee6a9b 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" @@ -11,7 +11,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, assert_equal, - bytes_to_hex_str, hex_str_to_bytes, sync_mempools, ) @@ -126,7 +125,7 @@ class ListTransactionsTest(BitcoinTestFramework): # 1. Chain a few transactions that don't opt-in. txid_1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - assert(not is_opt_in(self.nodes[0], txid_1)) + assert not is_opt_in(self.nodes[0], txid_1) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) sync_mempools(self.nodes) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) @@ -146,7 +145,7 @@ class ListTransactionsTest(BitcoinTestFramework): txid_2 = self.nodes[1].sendrawtransaction(tx2_signed) # ...and check the result - assert(not is_opt_in(self.nodes[1], txid_2)) + assert not is_opt_in(self.nodes[1], txid_2) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) sync_mempools(self.nodes) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) @@ -158,11 +157,11 @@ class ListTransactionsTest(BitcoinTestFramework): tx3 = self.nodes[0].createrawtransaction(inputs, outputs) tx3_modified = tx_from_hex(tx3) tx3_modified.vin[0].nSequence = 0 - tx3 = bytes_to_hex_str(tx3_modified.serialize()) + tx3 = tx3_modified.serialize().hex() tx3_signed = self.nodes[0].signrawtransactionwithwallet(tx3)['hex'] txid_3 = self.nodes[0].sendrawtransaction(tx3_signed) - assert(is_opt_in(self.nodes[0], txid_3)) + assert is_opt_in(self.nodes[0], txid_3) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) sync_mempools(self.nodes) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) @@ -176,7 +175,7 @@ class ListTransactionsTest(BitcoinTestFramework): tx4_signed = self.nodes[1].signrawtransactionwithwallet(tx4)["hex"] txid_4 = self.nodes[1].sendrawtransaction(tx4_signed) - assert(not is_opt_in(self.nodes[1], txid_4)) + assert not is_opt_in(self.nodes[1], txid_4) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) sync_mempools(self.nodes) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) @@ -184,10 +183,10 @@ class ListTransactionsTest(BitcoinTestFramework): # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee - tx3_b = bytes_to_hex_str(tx3_b.serialize()) + tx3_b = tx3_b.serialize().hex() tx3_b_signed = self.nodes[0].signrawtransactionwithwallet(tx3_b)['hex'] - txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, True) - assert(is_opt_in(self.nodes[0], txid_3b)) + txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, 0) + assert is_opt_in(self.nodes[0], txid_3b) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) sync_mempools(self.nodes) @@ -203,7 +202,7 @@ class ListTransactionsTest(BitcoinTestFramework): # After mining a transaction, it's no longer BIP125-replaceable self.nodes[0].generate(1) - assert(txid_3b not in self.nodes[0].getrawmempool()) + assert txid_3b not in self.nodes[0].getrawmempool() assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index df778f57df..984ffab5a4 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 00bf58d709..8aafa94c2e 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -2,31 +2,70 @@ # Copyright (c) 2017-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test resendwallettransactions RPC.""" +"""Test that the wallet resends transactions periodically.""" +from collections import defaultdict +import time +from test_framework.blocktools import create_block, create_coinbase +from test_framework.messages import ToHex +from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, wait_until + +class P2PStoreTxInvs(P2PInterface): + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == 1: + # save txid + self.tx_invs_received[i.hash] += 1 class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['--walletbroadcast=false']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): - # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + node = self.nodes[0] # alias + + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a new transaction and wait until it's broadcast") + txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + + # Can take a few seconds due to transaction trickling + wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + + # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a block") + # Create and submit a block without the transaction. + # Transactions are only rebroadcast if there has been a block at least five minutes + # after the last time we tried to broadcast. Use mocktime and give an extra minute to be sure. + block_time = int(time.time()) + 6 * 60 + node.setmocktime(block_time) + block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockchaininfo()['blocks']), block_time) + block.nVersion = 3 + block.rehash() + block.solve() + node.submitblock(ToHex(block)) - # Should return an empty array if there aren't unconfirmed wallet transactions. - self.stop_node(0) - self.start_node(0, extra_args=[]) - assert_equal(self.nodes[0].resendwallettransactions(), []) + # Transaction should not be rebroadcast + node.p2ps[1].sync_with_ping() + assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - # Should return an array with the unconfirmed wallet transaction. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - assert_equal(self.nodes[0].resendwallettransactions(), [txid]) + self.log.info("Transaction should be rebroadcast after 30 minutes") + # Use mocktime and give an extra 5 minutes to be sure. + rebroadcast_time = int(time.time()) + 41 * 60 + node.setmocktime(rebroadcast_time) + wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) if __name__ == '__main__': ResendWalletTransactionsTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 2f927502ff..af1b321a07 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index f114d5ab68..8dcfd74f56 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" @@ -34,6 +34,14 @@ class TxnMallTest(BitcoinTestFramework): def run_test(self): # All nodes should start with 1,250 BTC: starting_balance = 1250 + + # All nodes should be out of IBD. + # If the nodes are not all out of IBD, that can interfere with + # blockchain sync later in the test when nodes are connected, due to + # timing issues. + for n in self.nodes: + assert n.getblockchaininfo()["initialblockdownload"] == False + for i in range(4): assert_equal(self.nodes[i].getbalance(), starting_balance) self.nodes[i].getnewaddress("") # bug workaround, coins generated assigned to first getnewaddress! diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index c370ce0c04..3b05d5055c 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index f5d4780b6d..224e62f04a 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -39,7 +39,7 @@ def parse_function_calls(function_name, source_code): >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ - assert(type(function_name) is str and type(source_code) is str and function_name) + assert type(function_name) is str and type(source_code) is str and function_name lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] @@ -53,7 +53,7 @@ def normalize(s): >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ - assert(type(s) is str) + assert type(s) is str s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub("/\*.*?\*/", " ", s) @@ -77,7 +77,7 @@ def escape(s): >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s @@ -92,7 +92,7 @@ def unescape(s): >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s @@ -151,10 +151,10 @@ def parse_function_call_and_arguments(function_name, function_call): >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] """ - assert(type(function_name) is str and type(function_call) is str and function_name) + assert type(function_name) is str and type(function_call) is str and function_name remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) - assert(remaining.startswith(expected_function_call)) + assert remaining.startswith(expected_function_call) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 @@ -213,7 +213,7 @@ def parse_string_content(argument): >>> parse_string_content('1 2 3') '' """ - assert(type(argument) is str) + assert type(argument) is str string_content = "" in_string = False for char in normalize(escape(argument)): @@ -240,7 +240,7 @@ def count_format_specifiers(format_string): >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ - assert(type(format_string) is str) + assert type(format_string) is str format_string = format_string.replace('%%', 'X') n = 0 in_specifier = False @@ -262,27 +262,27 @@ def main(): parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " "argument (e.g. 1 in the case of fprintf)", default=0) parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", type=argparse.FileType("r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args() - exit_code = 0 - for f in args.file: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue sys.exit(exit_code) diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 1534d5ef68..2b6c78c2c8 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -8,6 +8,7 @@ KNOWN_VIOLATIONS=( "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/init.cpp:.*fprintf" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" @@ -18,7 +19,6 @@ KNOWN_VIOLATIONS=( "src/util/strencodings.cpp:.*strtoul" "src/util/strencodings.h:.*atoi" "src/util/system.cpp:.*atoi" - "src/util/system.cpp:.*fprintf" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index d55119b266..643272de52 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -15,6 +15,7 @@ unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/chacha20.cpp unsigned-integer-overflow:crypto/ctaes/ctaes.c +unsigned-integer-overflow:crypto/poly1305.cpp unsigned-integer-overflow:crypto/ripemd160.cpp unsigned-integer-overflow:crypto/sha1.cpp unsigned-integer-overflow:crypto/sha256.cpp |