diff options
225 files changed, 5656 insertions, 3057 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index acb54dd354..28c1814998 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,6 +4,8 @@ This issue tracker is only for technical issues related to bitcoin-core. General bitcoin questions and/or support requests and are best directed to the [Bitcoin StackExchange](https://bitcoin.stackexchange.com). +For reporting security issues, please read instructions at [https://bitcoincore.org/en/contact/](https://bitcoincore.org/en/contact/). + ### Describe the issue ### Can you reliably reproduce the issue? diff --git a/Makefile.am b/Makefile.am index 44fdf9c9fb..76eba51906 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,6 +39,11 @@ OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) +DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ + $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ + $(top_srcdir)/contrib/bitcoind.bash-completion \ + $(top_srcdir)/contrib/init \ + $(top_srcdir)/contrib/rpm BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py @@ -211,7 +216,7 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) @@ -49,9 +49,10 @@ lots of money. ### Automated Testing -Developers are strongly encouraged to write [unit tests](/doc/unit-tests.md) for new code, and to +Developers are strongly encouraged to write [unit tests](src/test/README.md) for new code, and to submit new unit tests for old code. Unit tests can be compiled and run -(assuming they weren't disabled in configure) with: `make check` +(assuming they weren't disabled in configure) with: `make check`. Further details on running +and extending unit tests can be found in [/src/test/README.md](/src/test/README.md). There are also [regression and integration tests](/qa) of the RPC interface, written in Python, that are run automatically on the build server. diff --git a/configure.ac b/configure.ac index 705327e816..319e1afe27 100644 --- a/configure.ac +++ b/configure.ac @@ -45,7 +45,6 @@ else CXXFLAGS_overridden=no fi AC_PROG_CXX -m4_ifdef([AC_PROG_OBJCXX],[AC_PROG_OBJCXX]) dnl By default, libtool for mingw refuses to link static libs into a dll for dnl fear of mixing pic/non-pic objects, and import/export complications. Since @@ -60,6 +59,15 @@ AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC +dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures +dnl that we get the same -std flags for both. +m4_ifdef([AC_PROG_OBJCXX],[ +if test "x${OBJCXX+set}" = "x"; then + OBJCXX="${CXX}" +fi +AC_PROG_OBJCXX +]) + dnl Libtool init checks. LT_INIT([pic-only]) @@ -1126,3 +1134,28 @@ case ${OS} in mv qa/pull-tester/tests_config-2.py qa/pull-tester/tests_config.py ;; esac + +echo +echo "Options used to compile and link:" +echo " with wallet = $enable_wallet" +echo " with gui / qt = $bitcoin_enable_qt" +if test x$bitcoin_enable_qt != xno; then + echo " qt version = $bitcoin_qt_got_major_vers" + echo " with qr = $use_qr" +fi +echo " with zmq = $use_zmq" +echo " with test = $use_tests" +echo " with bench = $use_bench" +echo " with upnp = $use_upnp" +echo " debug enabled = $enable_debug" +echo +echo " target os = $TARGET_OS" +echo " build os = $BUILD_OS" +echo +echo " CC = $CC" +echo " CFLAGS = $CFLAGS" +echo " CPPFLAGS = $CPPFLAGS" +echo " CXX = $CXX" +echo " CXXFLAGS = $CXXFLAGS" +echo " LDFLAGS = $LDFLAGS" +echo diff --git a/contrib/README.md b/contrib/README.md index ab5f57587e..4ea9700f59 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -38,7 +38,7 @@ Scripts and notes for Mac builds. RPM spec file for building bitcoin-core on RPM based distributions ### [Gitian-build](/contrib/gitian-build.sh) ### -Script for running full gitian builds. +Script for running full Gitian builds. Test and Verify Tools --------------------- diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 5cac6e533c..0fa06f1aa9 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -47,7 +47,10 @@ Comment: Site: https://github.com/stephenhutchings/typicons.font Files: src/qt/res/icons/connect*.png src/qt/res/src/connect-*.svg + src/qt/res/icons/network_disabled.png + src/qt/res/src/network_disabled.svg Copyright: Marco Falke + Luke Dashjr License: Expat Comment: Inspired by Stephan Hutchings Typicons diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 60fe69e7e3..6c0047833f 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -24,21 +24,64 @@ the script should be called from the git root folder as follows. ``` git diff -U0 HEAD~1.. | ./contrib/devtools/clang-format-diff.py -p1 -i -v ``` +copyright\_header.py +==================== -fix-copyright-headers.py -======================== +Provides utilities for managing copyright headers of `The Bitcoin Core +developers` in repository source files. It has three subcommands: -Every year newly updated files need to have its copyright headers updated to reflect the current year. -If you run this script from the root folder it will automatically update the year on the copyright header for all -source files if these have a git commit from the current year. +``` +$ ./copyright_header.py report <base_directory> [verbose] +$ ./copyright_header.py update <base_directory> +$ ./copyright_header.py insert <file> +``` +Running these subcommands without arguments displays a usage string. -For example a file changed in 2015 (with 2015 being the current year): +copyright\_header.py report \<base\_directory\> [verbose] +--------------------------------------------------------- -```// Copyright (c) 2009-2013 The Bitcoin Core developers``` +Produces a report of all copyright header notices found inside the source files +of a repository. Useful to quickly visualize the state of the headers. +Specifying `verbose` will list the full filenames of files of each category. -would be changed to: +copyright\_header.py update \<base\_directory\> [verbose] +--------------------------------------------------------- +Updates all the copyright headers of `The Bitcoin Core developers` which were +changed in a year more recent than is listed. For example: +``` +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers +``` +where `<lastModifiedYear>` is obtained from the `git log` history. -```// Copyright (c) 2009-2015 The Bitcoin Core developers``` +This subcommand also handles copyright headers that have only a single year. In +those cases: +``` +// Copyright (c) <year> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers +``` +where the update is appropriate. + +copyright\_header.py insert \<file\> +------------------------------------ +Inserts a copyright header for `The Bitcoin Core developers` at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has `#!` starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be `<year_introduced>-<current_year>` where +`<year_introduced>` is according to the `git log` history. If +`<year_introduced>` is equal to `<current_year>`, it will be set as a single +year rather than two hyphenated years. + +If the file already has a copyright for `The Bitcoin Core developers`, the +script will exit. gen-manpages.sh =============== diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py new file mode 100755 index 0000000000..9f35c378bf --- /dev/null +++ b/contrib/devtools/copyright_header.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import re +import fnmatch +import sys +import subprocess +import datetime +import os + +################################################################################ +# file filtering +################################################################################ + +EXCLUDE = [ + # libsecp256k1: + 'src/secp256k1/include/secp256k1.h', + 'src/secp256k1/include/secp256k1_ecdh.h', + 'src/secp256k1/include/secp256k1_recovery.h', + 'src/secp256k1/include/secp256k1_schnorr.h', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h', + # auto generated: + 'src/univalue/lib/univalue_escapes.h', + 'src/qt/bitcoinstrings.cpp', + 'src/chainparamsseeds.h', + # other external copyrights: + 'src/tinyformat.h', + 'src/leveldb/util/env_win.cc', + 'src/crypto/ctaes/bench.c', + 'qa/rpc-tests/test_framework/bignum.py', + # python init: + '*__init__.py', +] +EXCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in EXCLUDE])) + +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] +INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) + +def applies_to_file(filename): + return ((EXCLUDE_COMPILED.match(filename) is None) and + (INCLUDE_COMPILED.match(filename) is not None)) + +################################################################################ +# obtain list of files in repo according to INCLUDE and EXCLUDE +################################################################################ + +GIT_LS_CMD = 'git ls-files' + +def call_git_ls(): + out = subprocess.check_output(GIT_LS_CMD.split(' ')) + return [f for f in out.decode("utf-8").split('\n') if f != ''] + +def get_filenames_to_examine(): + filenames = call_git_ls() + return sorted([filename for filename in filenames if + applies_to_file(filename)]) + +################################################################################ +# define and compile regexes for the patterns we are looking for +################################################################################ + + +COPYRIGHT_WITH_C = 'Copyright \(c\)' +COPYRIGHT_WITHOUT_C = 'Copyright' +ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) + +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +YEAR_LIST = '(%s)(, %s)+' % (YEAR, YEAR) +ANY_YEAR_STYLE = '(%s|%s)' % (YEAR_RANGE, YEAR_LIST) +ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE, + ANY_YEAR_STYLE)) + +ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) + +def compile_copyright_regex(copyright_style, year_style, name): + return re.compile('%s %s %s' % (copyright_style, year_style, name)) + +EXPECTED_HOLDER_NAMES = [ + "Satoshi Nakamoto\n", + "The Bitcoin Core developers\n", + "The Bitcoin Core developers \n", + "Bitcoin Core Developers\n", + "the Bitcoin Core developers\n", + "The Bitcoin developers\n", + "The LevelDB Authors\. All rights reserved\.\n", + "BitPay Inc\.\n", + "BitPay, Inc\.\n", + "University of Illinois at Urbana-Champaign\.\n", + "MarcoFalke\n", + "Pieter Wuille\n", + "Pieter Wuille +\*\n", + "Pieter Wuille, Gregory Maxwell +\*\n", + "Pieter Wuille, Andrew Poelstra +\*\n", + "Andrew Poelstra +\*\n", + "Wladimir J. van der Laan\n", + "Jeff Garzik\n", + "Diederik Huys, Pieter Wuille +\*\n", + "Thomas Daede, Cory Fields +\*\n", + "Jan-Klaas Kollhof\n", + "Sam Rushing\n", + "ArtForz -- public domain half-a-node\n", +] + +DOMINANT_STYLE_COMPILED = {} +YEAR_LIST_STYLE_COMPILED = {} +WITHOUT_C_STYLE_COMPILED = {} + +for holder_name in EXPECTED_HOLDER_NAMES: + DOMINANT_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) + YEAR_LIST_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) + WITHOUT_C_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, + holder_name)) + +################################################################################ +# search file contents for copyright message of particular category +################################################################################ + +def get_count_of_copyrights_of_any_style_any_holder(contents): + return len(ANY_COPYRIGHT_COMPILED.findall(contents)) + +def file_has_dominant_style_copyright_for_holder(contents, holder_name): + match = DOMINANT_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_year_list_style_copyright_for_holder(contents, holder_name): + match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_without_c_style_copyright_for_holder(contents, holder_name): + match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +################################################################################ +# get file info +################################################################################ + +def read_file(filename): + return open(os.path.abspath(filename), 'r').read() + +def gather_file_info(filename): + info = {} + info['filename'] = filename + c = read_file(filename) + info['contents'] = c + + info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c) + + info['classified_copyrights'] = 0 + info['dominant_style'] = {} + info['year_list_style'] = {} + info['without_c_style'] = {} + for holder_name in EXPECTED_HOLDER_NAMES: + has_dominant_style = ( + file_has_dominant_style_copyright_for_holder(c, holder_name)) + has_year_list_style = ( + file_has_year_list_style_copyright_for_holder(c, holder_name)) + has_without_c_style = ( + file_has_without_c_style_copyright_for_holder(c, holder_name)) + info['dominant_style'][holder_name] = has_dominant_style + info['year_list_style'][holder_name] = has_year_list_style + info['without_c_style'][holder_name] = has_without_c_style + if has_dominant_style or has_year_list_style or has_without_c_style: + info['classified_copyrights'] = info['classified_copyrights'] + 1 + return info + +################################################################################ +# report execution +################################################################################ + +SEPARATOR = '-'.join(['' for _ in range(80)]) + +def print_filenames(filenames, verbose): + if not verbose: + return + for filename in filenames: + print("\t%s" % filename) + +def print_report(file_infos, verbose): + print(SEPARATOR) + examined = [i['filename'] for i in file_infos] + print("%d files examined according to INCLUDE and EXCLUDE fnmatch rules" % + len(examined)) + print_filenames(examined, verbose) + + print(SEPARATOR) + print('') + zero_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 0] + print("%4d with zero copyrights" % len(zero_copyrights)) + print_filenames(zero_copyrights, verbose) + one_copyright = [i['filename'] for i in file_infos if + i['all_copyrights'] == 1] + print("%4d with one copyright" % len(one_copyright)) + print_filenames(one_copyright, verbose) + two_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 2] + print("%4d with two copyrights" % len(two_copyrights)) + print_filenames(two_copyrights, verbose) + three_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 3] + print("%4d with three copyrights" % len(three_copyrights)) + print_filenames(three_copyrights, verbose) + four_or_more_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] >= 4] + print("%4d with four or more copyrights" % len(four_or_more_copyrights)) + print_filenames(four_or_more_copyrights, verbose) + print('') + print(SEPARATOR) + print('Copyrights with dominant style:\ne.g. "Copyright (c)" and ' + '"<year>" or "<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + dominant_style = [i['filename'] for i in file_infos if + i['dominant_style'][holder_name]] + if len(dominant_style) > 0: + print("%4d with '%s'" % (len(dominant_style), + holder_name.replace('\n', '\\n'))) + print_filenames(dominant_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with year list style:\ne.g. "Copyright (c)" and ' + '"<year1>, <year2>, ...":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + year_list_style = [i['filename'] for i in file_infos if + i['year_list_style'][holder_name]] + if len(year_list_style) > 0: + print("%4d with '%s'" % (len(year_list_style), + holder_name.replace('\n', '\\n'))) + print_filenames(year_list_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "<year>" or ' + '"<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + without_c_style = [i['filename'] for i in file_infos if + i['without_c_style'][holder_name]] + if len(without_c_style) > 0: + print("%4d with '%s'" % (len(without_c_style), + holder_name.replace('\n', '\\n'))) + print_filenames(without_c_style, verbose) + + print('') + print(SEPARATOR) + + unclassified_copyrights = [i['filename'] for i in file_infos if + i['classified_copyrights'] < i['all_copyrights']] + print("%d with unexpected copyright holder names" % + len(unclassified_copyrights)) + print_filenames(unclassified_copyrights, verbose) + print(SEPARATOR) + +def exec_report(base_directory, verbose): + original_cwd = os.getcwd() + os.chdir(base_directory) + filenames = get_filenames_to_examine() + file_infos = [gather_file_info(f) for f in filenames] + print_report(file_infos, verbose) + os.chdir(original_cwd) + +################################################################################ +# report cmd +################################################################################ + +REPORT_USAGE = """ +Produces a report of all copyright header notices found inside the source files +of a repository. + +Usage: + $ ./copyright_header.py report <base_directory> [verbose] + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. + [verbose] - Includes a list of every file of each subcategory in the report. +""" + +def report_cmd(argv): + if len(argv) == 2: + sys.exit(REPORT_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad <base_directory>: %s" % base_directory) + + if len(argv) == 3: + verbose = False + elif argv[3] == 'verbose': + verbose = True + else: + sys.exit("*** unknown argument: %s" % argv[2]) + + exec_report(base_directory, verbose) + +################################################################################ +# query git for year of last change +################################################################################ + +GIT_LOG_CMD = "git log --pretty=format:%%ai %s" + +def call_git_log(filename): + out = subprocess.check_output((GIT_LOG_CMD % filename).split(' ')) + return out.decode("utf-8").split('\n') + +def get_git_change_years(filename): + git_log_lines = call_git_log(filename) + if len(git_log_lines) == 0: + return [datetime.date.today().year] + # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" + return [line.split(' ')[0].split('-')[0] for line in git_log_lines] + +def get_most_recent_git_change_year(filename): + return max(get_git_change_years(filename)) + +################################################################################ +# read and write to file +################################################################################ + +def read_file_lines(filename): + f = open(os.path.abspath(filename), 'r') + file_lines = f.readlines() + f.close() + return file_lines + +def write_file_lines(filename, file_lines): + f = open(os.path.abspath(filename), 'w') + f.write(''.join(file_lines)) + f.close() + +################################################################################ +# update header years execution +################################################################################ + +COPYRIGHT = 'Copyright \(c\)' +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +HOLDER = 'The Bitcoin Core developers' +UPDATEABLE_LINE_COMPILED = re.compile(' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) + +def get_updatable_copyright_line(file_lines): + index = 0 + for line in file_lines: + if UPDATEABLE_LINE_COMPILED.search(line) is not None: + return index, line + index = index + 1 + return None, None + +def parse_year_range(year_range): + year_split = year_range.split('-') + start_year = year_split[0] + if len(year_split) == 1: + return start_year, start_year + return start_year, year_split[1] + +def year_range_to_str(start_year, end_year): + if start_year == end_year: + return start_year + return "%s-%s" % (start_year, end_year) + +def create_updated_copyright_line(line, last_git_change_year): + copyright_splitter = 'Copyright (c) ' + copyright_split = line.split(copyright_splitter) + # Preserve characters on line that are ahead of the start of the copyright + # notice - they are part of the comment block and vary from file-to-file. + before_copyright = copyright_split[0] + after_copyright = copyright_split[1] + + space_split = after_copyright.split(' ') + year_range = space_split[0] + start_year, end_year = parse_year_range(year_range) + if end_year == last_git_change_year: + return line + return (before_copyright + copyright_splitter + + year_range_to_str(start_year, last_git_change_year) + ' ' + + ' '.join(space_split[1:])) + +def update_updatable_copyright(filename): + file_lines = read_file_lines(filename) + index, line = get_updatable_copyright_line(file_lines) + if not line: + print_file_action_message(filename, "No updatable copyright.") + return + last_git_change_year = get_most_recent_git_change_year(filename) + new_line = create_updated_copyright_line(line, last_git_change_year) + if line == new_line: + print_file_action_message(filename, "Copyright up-to-date.") + return + file_lines[index] = new_line + write_file_lines(filename, file_lines) + print_file_action_message(filename, + "Copyright updated! -> %s" % last_git_change_year) + +def exec_update_header_year(base_directory): + original_cwd = os.getcwd() + os.chdir(base_directory) + for filename in get_filenames_to_examine(): + update_updatable_copyright(filename) + os.chdir(original_cwd) + +################################################################################ +# update cmd +################################################################################ + +UPDATE_USAGE = """ +Updates all the copyright headers of "The Bitcoin Core developers" which were +changed in a year more recent than is listed. For example: + +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers + +where <lastModifiedYear> is obtained from the 'git log' history. + +This subcommand also handles copyright headers that have only a single year. In those cases: + +// Copyright (c) <year> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers + +where the update is appropriate. + +Usage: + $ ./copyright_header.py update <base_directory> + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. +""" + +def print_file_action_message(filename, action): + print("%-52s %s" % (filename, action)) + +def update_cmd(argv): + if len(argv) != 3: + sys.exit(UPDATE_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad base_directory: %s" % base_directory) + exec_update_header_year(base_directory) + +################################################################################ +# inserted copyright header format +################################################################################ + +def get_header_lines(header, start_year, end_year): + lines = header.split('\n')[1:-1] + lines[0] = lines[0] % year_range_to_str(start_year, end_year) + return [line + '\n' for line in lines] + +CPP_HEADER = ''' +// Copyright (c) %s The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_cpp_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) + +PYTHON_HEADER = ''' +# Copyright (c) %s The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_python_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) + +################################################################################ +# query git for year of last change +################################################################################ + +def get_git_change_year_range(filename): + years = get_git_change_years(filename) + return min(years), max(years) + +################################################################################ +# check for existing core copyright +################################################################################ + +def file_already_has_core_copyright(file_lines): + index, _ = get_updatable_copyright_line(file_lines) + return index != None + +################################################################################ +# insert header execution +################################################################################ + +def file_has_hashbang(file_lines): + if len(file_lines) < 1: + return False + if len(file_lines[0]) <= 2: + return False + return file_lines[0][:2] == '#!' + +def insert_python_header(filename, file_lines, start_year, end_year): + if file_has_hashbang(file_lines): + insert_idx = 1 + else: + insert_idx = 0 + header_lines = get_python_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(insert_idx, line) + write_file_lines(filename, file_lines) + +def insert_cpp_header(filename, file_lines, start_year, end_year): + header_lines = get_cpp_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(0, line) + write_file_lines(filename, file_lines) + +def exec_insert_header(filename, style): + file_lines = read_file_lines(filename) + if file_already_has_core_copyright(file_lines): + sys.exit('*** %s already has a copyright by The Bitcoin Core developers' + % (filename)) + start_year, end_year = get_git_change_year_range(filename) + if style == 'python': + insert_python_header(filename, file_lines, start_year, end_year) + else: + insert_cpp_header(filename, file_lines, start_year, end_year) + +################################################################################ +# insert cmd +################################################################################ + +INSERT_USAGE = """ +Inserts a copyright header for "The Bitcoin Core developers" at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has a '#!' starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be: + +"<year_introduced>-<current_year>" + +where <year_introduced> is according to the 'git log' history. If +<year_introduced> is equal to <current_year>, the date will be set to be: + +"<current_year>" + +If the file already has a copyright for "The Bitcoin Core developers", the +script will exit. + +Usage: + $ ./copyright_header.py insert <file> + +Arguments: + <file> - A source file in the bitcoin repository. +""" + +def insert_cmd(argv): + if len(argv) != 3: + sys.exit(INSERT_USAGE) + + filename = argv[2] + if not os.path.isfile(filename): + sys.exit("*** bad filename: %s" % filename) + _, extension = os.path.splitext(filename) + if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: + sys.exit("*** cannot insert for file extension %s" % extension) + + if extension == '.py': + style = 'python' + else: + style = 'cpp' + exec_insert_header(filename, style) + +################################################################################ +# UI +################################################################################ + +USAGE = """ +copyright_header.py - utilities for managing copyright headers of 'The Bitcoin +Core developers' in repository source files. + +Usage: + $ ./copyright_header <subcommand> + +Subcommands: + report + update + insert + +To see subcommand usage, run them without arguments. +""" + +SUBCOMMANDS = ['report', 'update', 'insert'] + +if __name__ == "__main__": + if len(sys.argv) == 1: + sys.exit(USAGE) + subcommand = sys.argv[1] + if subcommand not in SUBCOMMANDS: + sys.exit(USAGE) + if subcommand == 'report': + report_cmd(sys.argv) + elif subcommand == 'update': + update_cmd(sys.argv) + elif subcommand == 'insert': + insert_cmd(sys.argv) diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py deleted file mode 100755 index 54836bd83f..0000000000 --- a/contrib/devtools/fix-copyright-headers.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -""" -Run this script to update all the copyright headers of files -that were changed this year. - -For example: - -// Copyright (c) 2009-2012 The Bitcoin Core developers - -it will change it to - -// Copyright (c) 2009-2015 The Bitcoin Core developers -""" -import subprocess -import time -import re - -CMD_GIT_LIST_FILES = ['git', 'ls-files'] -CMD_GIT_DATE = ['git', 'log', '--format=%ad', '--date=short', '-1'] -CMD_PERL_REGEX = ['perl', '-pi', '-e'] -REGEX_TEMPLATE = 's/(20\\d\\d)(?:-20\\d\\d)? The Bitcoin/$1-%s The Bitcoin/' - -FOLDERS = ["qa/", "src/"] -EXTENSIONS = [".cpp",".h", ".py"] - - -def get_git_date(file_path): - d = subprocess.run(CMD_GIT_DATE + [file_path], - stdout=subprocess.PIPE, - check=True, - universal_newlines=True).stdout - # yyyy-mm-dd - return d.split('-')[0] - - -def skip_file(file_path): - for ext in EXTENSIONS: - if file_path.endswith(ext): - return False - else: - return True - -if __name__ == "__main__": - year = str(time.gmtime()[0]) - regex_current = re.compile("%s The Bitcoin" % year) - n = 1 - for folder in FOLDERS: - for file_path in subprocess.run( - CMD_GIT_LIST_FILES + [folder], - stdout=subprocess.PIPE, - check=True, - universal_newlines=True - ).stdout.split("\n"): - if skip_file(file_path): - # print(file_path, "(skip)") - continue - git_date = get_git_date(file_path) - if not year == git_date: - # print(file_path, year, "(skip)") - continue - if regex_current.search(open(file_path, "r").read()) is not None: - # already up to date - # print(file_path, year, "(skip)") - continue - print(n, file_path, "(update to %s)" % year) - subprocess.run(CMD_PERL_REGEX + [REGEX_TEMPLATE % year, file_path], check=True) - n = n + 1 diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index f82362fe41..aae966a8f6 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 Bitcoin Core Developers +# Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 32b57b3160..fe01b5b957 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -27,7 +27,7 @@ remotes: files: [] script: | WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 985649619f..4e58bec74e 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=10.7 +OSX_MIN_VERSION=10.8 OSX_SDK_VERSION=10.11 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk LD64_VERSION=253.9 diff --git a/doc/Doxyfile b/doc/Doxyfile index a0cbf7139a..ef55acdbc3 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = Bitcoin +PROJECT_NAME = "Bitcoin Core" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version diff --git a/doc/README.md b/doc/README.md index e4fa49614a..8b9c0ea262 100644 --- a/doc/README.md +++ b/doc/README.md @@ -53,7 +53,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) -- [Unit Tests](unit-tests.md) - [Travis CI](travis-ci.md) - [Unauthenticated REST Interface](REST-interface.md) - [Shared Libraries](shared-libraries.md) diff --git a/doc/build-osx.md b/doc/build-osx.md index bc90a30562..63a7ee28ca 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -90,6 +90,6 @@ Uncheck everything except Qt Creator during the installation process. Notes ----- -* Tested on OS X 10.7 through 10.11 on 64-bit Intel processors only. +* Tested on OS X 10.8 through 10.12 on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) diff --git a/doc/build-unix.md b/doc/build-unix.md index ba7b9cd18d..5b0a38457e 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -96,13 +96,13 @@ pass `--with-incompatible-bdb` to configure. See the section "Disable-wallet mode" to build Bitcoin Core without wallet. -Optional: +Optional (see --with-miniupnpc and --enable-upnp-default): - sudo apt-get install libminiupnpc-dev (see --with-miniupnpc and --enable-upnp-default) + sudo apt-get install libminiupnpc-dev -ZMQ dependencies: +ZMQ dependencies (provides ZMQ API 4.x): - sudo apt-get install libzmq3-dev (provides ZMQ API 4.x) + sudo apt-get install libzmq3-dev Dependencies for the GUI: Ubuntu & Debian ----------------------------------------- diff --git a/doc/build-windows.md b/doc/build-windows.md index 044356830a..b6a047b9c3 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -7,16 +7,18 @@ Most developers use cross-compilation from Ubuntu to build executables for Windows. This is also used to build the release binaries. While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64), -using the Windows Subsystem For Linux is the most straight forward. If you are building with -an alternative method, please contribute the instructions here for others who are running versions +using the Windows Subsystem For Linux is the most straightforward. If you are building with +another method, please contribute the instructions here for others who are running versions of Windows that are not compatible with the Windows Subsystem for Linux. -Compiling with the Windows Subsystem For Linux -------------------- +Compiling with Windows Subsystem For Linux +------------------------------------------- -With Windows 10, Microsoft has released a new feature named the -[Windows Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu based -environment. Within this environment you can cross compile for Windows without the need for a separate Linux VM or Server. +With Windows 10, Microsoft has released a new feature named the [Windows +Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This +feature allows you to run a bash shell directly on Windows in an Ubuntu based +environment. Within this environment you can cross compile for Windows without +the need for a separate Linux VM or Server. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. @@ -37,9 +39,6 @@ To get the bash shell, you must first activate the feature in Windows. * Create a new UNIX user account (this is a separate account from your Windows account) After the bash shell is active, you can follow the instructions below for Windows 64-bit Cross-compilation. -When building dependencies within the 'depends' folder, you may encounter an error building -the protobuf dependency. If this occurs, re-run the command with sudo. This is likely -a bug with the Windows Subsystem for Linux feature and may be fixed with a future update. Cross-compilation ------------------- @@ -48,28 +47,55 @@ These steps can be performed on, for example, an Ubuntu VM. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. -Make sure you install the build requirements mentioned in -[build-unix.md](/doc/build-unix.md). -Then, install the toolchains and curl: +First, install the general dependencies: + + sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl + +A host toolchain (`build-essential`) is necessary because some dependency +packages (such as `protobuf`) need to build host utilities that are used in the +build process. + +## Building for 64-bit Windows + +To build executables for Windows 64-bit, install the following dependencies: - sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev g++-mingw-w64-x86-64 mingw-w64-x86-64-dev curl + sudo apt-get install g++-mingw-w64-x86-64 mingw-w64-x86-64-dev -To build executables for Windows 32-bit: +Then build using: cd depends - make HOST=i686-w64-mingw32 -j4 + make HOST=x86_64-w64-mingw32 -j4 cd .. ./autogen.sh # not required when building from tarball - ./configure --prefix=`pwd`/depends/i686-w64-mingw32 + CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make -To build executables for Windows 64-bit: +## Building for 32-bit Windows + +To build executables for Windows 32-bit, install the following dependencies: + + sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev + +Then build using: cd depends - make HOST=x86_64-w64-mingw32 -j4 + make HOST=i686-w64-mingw32 -j4 cd .. ./autogen.sh # not required when building from tarball - ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + CONFIG_SITE=$PWD/depends/i686-w64-mingw32/share/config.site ./configure --prefix=/ make +## Depends system + For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. + +Installation +------------- + +After building using the Windows subsystem it can be useful to copy the compiled +executables to a directory on the windows drive in the same directory structure +as they appear in the release `.zip` archive. This can be done in the following +way. This will install to `c:\workspace\bitcoin`, for example: + + make install DESTDIR=/mnt/c/workspace/bitcoin + diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 70c0690ba3..b0794e6d30 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -331,6 +331,32 @@ Strings and formatting - *Rationale*: Bitcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion +Variable names +-------------- + +The shadowing warning (`-Wshadow`) is enabled by default. It prevents issues rising +from using a different variable with the same name. + +Please name variables so that their names do not shadow variables defined in the source code. + +E.g. in member initializers, prepend `_` to the argument name shadowing the +member name: + +```c++ +class AddressBookPage +{ + Mode mode; +} + +AddressBookPage::AddressBookPage(Mode _mode) : + mode(_mode) +... +``` + +When using nested cycles, do not name the inner cycle variable the same as in +upper cycle etc. + + Threads and synchronization ---------------------------- diff --git a/doc/files.md b/doc/files.md index f7eca57dcb..928977143b 100644 --- a/doc/files.md +++ b/doc/files.md @@ -10,6 +10,7 @@ * db.log: wallet database log file * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 +* mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 * wallet.dat: personal wallet (BDB) with keys and transactions * .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown): since 0.12.0 diff --git a/doc/release-notes.md b/doc/release-notes.md index 0463cb8a61..fe7f69d1f8 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -48,6 +48,19 @@ Low-level RPC changes an optional third arg, which was always ignored. Make sure to never pass more than two arguments. +Removal of Priority Estimation +------------------------------ + +- Estimation of "priority" needed for a transaction to be included within a target + number of blocks has been removed. The rpc calls are deprecated and will either + return -1 or 1e24 appropriately. The format for `fee_estimates.dat` has also + changed to no longer save these priority estimates. It will automatically be + converted to the new format which is not readable by prior versions of the + software. + +- The concept of "priority" transactions is planned to be removed in the next + major version. To prepare for this, the default for the rate limit of priority + transactions (`-limitfreerelay`) has been set to `0` kB/minute. 0.14.0 Change log ================= diff --git a/doc/release-notes/release-notes-0.13.1.md b/doc/release-notes/release-notes-0.13.1.md new file mode 100644 index 0000000000..75c2d61be8 --- /dev/null +++ b/doc/release-notes/release-notes-0.13.1.md @@ -0,0 +1,410 @@ +Bitcoin Core version 0.13.1 is now available from: + + <https://bitcoin.org/bin/bitcoin-core-0.13.1/> + +This is a new minor version release, including activation parameters for the +segwit softfork, various bugfixes and performance improvements, as well as +updated translations. + +Please report bugs using the issue tracker at github: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +Compatibility +============== + +Microsoft ended support for Windows XP on [April 8th, 2014](https://www.microsoft.com/en-us/WindowsForBusiness/end-of-xp-support), +an OS initially released in 2001. This means that not even critical security +updates will be released anymore. Without security updates, using a bitcoin +wallet on a XP machine is irresponsible at least. + +In addition to that, with 0.12.x there have been varied reports of Bitcoin Core +randomly crashing on Windows XP. It is [not clear](https://github.com/bitcoin/bitcoin/issues/7681#issuecomment-217439891) +what the source of these crashes is, but it is likely that upstream +libraries such as Qt are no longer being tested on XP. + +We do not have time nor resources to provide support for an OS that is +end-of-life. From 0.13.0 on, Windows XP is no longer supported. Users are +suggested to upgrade to a newer version of Windows, or install an alternative OS +that is supported. + +No attempt is made to prevent installing or running the software on Windows XP, +you can still do so at your own risk, but do not expect it to work: do not +report issues about Windows XP to the issue tracker. + +From 0.13.1 onwards OS X 10.7 is no longer supported. 0.13.0 was intended to work on 10.7+, +but severe issues with the libc++ version on 10.7.x keep it from running reliably. +0.13.1 now requires 10.8+, and will communicate that to 10.7 users, rather than crashing unexpectedly. + +Notable changes +=============== + +Segregated witness soft fork +---------------------------- + +Segregated witness (segwit) is a soft fork that, if activated, will +allow transaction-producing software to separate (segregate) transaction +signatures (witnesses) from the part of the data in a transaction that is +covered by the txid. This provides several immediate benefits: + +- **Elimination of unwanted transaction malleability:** Segregating the witness + allows both existing and upgraded software to calculate the transaction + identifier (txid) of transactions without referencing the witness, which can + sometimes be changed by third-parties (such as miners) or by co-signers in a + multisig spend. This solves all known cases of unwanted transaction + malleability, which is a problem that makes programming Bitcoin wallet + software more difficult and which seriously complicates the design of smart + contracts for Bitcoin. + +- **Capacity increase:** Segwit transactions contain new fields that are not + part of the data currently used to calculate the size of a block, which + allows a block containing segwit transactions to hold more data than allowed + by the current maximum block size. Estimates based on the transactions + currently found in blocks indicate that if all wallets switch to using + segwit, the network will be able to support about 70% more transactions. The + network will also be able to support more of the advanced-style payments + (such as multisig) than it can support now because of the different weighting + given to different parts of a transaction after segwit activates (see the + following section for details). + +- **Weighting data based on how it affects node performance:** Some parts of + each Bitcoin block need to be stored by nodes in order to validate future + blocks; other parts of a block can be immediately forgotten (pruned) or used + only for helping other nodes sync their copy of the block chain. One large + part of the immediately prunable data are transaction signatures (witnesses), + and segwit makes it possible to give a different "weight" to segregated + witnesses to correspond with the lower demands they place on node resources. + Specifically, each byte of a segregated witness is given a weight of 1, each + other byte in a block is given a weight of 4, and the maximum allowed weight + of a block is 4 million. Weighting the data this way better aligns the most + profitable strategy for creating blocks with the long-term costs of block + validation. + +- **Signature covers value:** A simple improvement in the way signatures are + generated in segwit simplifies the design of secure signature generators + (such as hardware wallets), reduces the amount of data the signature + generator needs to download, and allows the signature generator to operate + more quickly. This is made possible by having the generator sign the amount + of bitcoins they think they are spending, and by having full nodes refuse to + accept those signatures unless the amount of bitcoins being spent is exactly + the same as was signed. For non-segwit transactions, wallets instead had to + download the complete previous transactions being spent for every payment + they made, which could be a slow operation on hardware wallets and in other + situations where bandwidth or computation speed was constrained. + +- **Linear scaling of sighash operations:** In 2015 a block was produced that + required about 25 seconds to validate on modern hardware because of the way + transaction signature hashes are performed. Other similar blocks, or blocks + that could take even longer to validate, can still be produced today. The + problem that caused this can't be fixed in a soft fork without unwanted + side-effects, but transactions that opt-in to using segwit will now use a + different signature method that doesn't suffer from this problem and doesn't + have any unwanted side-effects. + +- **Increased security for multisig:** Bitcoin addresses (both P2PKH addresses + that start with a '1' and P2SH addresses that start with a '3') use a hash + function known as RIPEMD-160. For P2PKH addresses, this provides about 160 + bits of security---which is beyond what cryptographers believe can be broken + today. But because P2SH is more flexible, only about 80 bits of security is + provided per address. Although 80 bits is very strong security, it is within + the realm of possibility that it can be broken by a powerful adversary. + Segwit allows advanced transactions to use the SHA256 hash function instead, + which provides about 128 bits of security (that is 281 trillion times as + much security as 80 bits and is equivalent to the maximum bits of security + believed to be provided by Bitcoin's choice of parameters for its Elliptic + Curve Digital Security Algorithm [ECDSA].) + +- **More efficient almost-full-node security** Satoshi Nakamoto's original + Bitcoin paper describes a method for allowing newly-started full nodes to + skip downloading and validating some data from historic blocks that are + protected by large amounts of proof of work. Unfortunately, Nakamoto's + method can't guarantee that a newly-started node using this method will + produce an accurate copy of Bitcoin's current ledger (called the UTXO set), + making the node vulnerable to falling out of consensus with other nodes. + Although the problems with Nakamoto's method can't be fixed in a soft fork, + Segwit accomplishes something similar to his original proposal: it makes it + possible for a node to optionally skip downloading some blockchain data + (specifically, the segregated witnesses) while still ensuring that the node + can build an accurate copy of the UTXO set for the block chain with the most + proof of work. Segwit enables this capability at the consensus layer, but + note that Bitcoin Core does not provide an option to use this capability as + of this 0.13.1 release. + +- **Script versioning:** Segwit makes it easy for future soft forks to allow + Bitcoin users to individually opt-in to almost any change in the Bitcoin + Script language when those users receive new transactions. Features + currently being researched by Bitcoin Core contributors that may use this + capability include support for Schnorr signatures, which can improve the + privacy and efficiency of multisig transactions (or transactions with + multiple inputs), and Merklized Abstract Syntax Trees (MAST), which can + improve the privacy and efficiency of scripts with two or more conditions. + Other Bitcoin community members are studying several other improvements + that can be made using script versioning. + +Activation for the segwit soft fork is being managed using BIP9 +versionbits. Segwit's version bit is bit 1, and nodes will begin +tracking which blocks signal support for segwit at the beginning of the +first retarget period after segwit's start date of 15 November 2016. If +95% of blocks within a 2,016-block retarget period (about two weeks) +signal support for segwit, the soft fork will be locked in. After +another 2,016 blocks, segwit will activate. + +For more information about segwit, please see the [segwit FAQ][], the +[segwit wallet developers guide][] or BIPs [141][BIP141], [143][BIP143], +[144][BIP144], and [145][BIP145]. If you're a miner or mining pool +operator, please see the [versionbits FAQ][] for information about +signaling support for a soft fork. + +[Segwit FAQ]: https://bitcoincore.org/en/2016/01/26/segwit-benefits/ +[segwit wallet developers guide]: https://bitcoincore.org/en/segwit_wallet_dev/ +[BIP141]: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki +[BIP143]: https://github.com/bitcoin/bips/blob/master/bip-0143.mediawiki +[BIP144]: https://github.com/bitcoin/bips/blob/master/bip-0144.mediawiki +[BIP145]: https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki +[versionbits FAQ]: https://bitcoincore.org/en/2016/06/08/version-bits-miners-faq/ + + +Null dummy soft fork +------------------- + +Combined with the segwit soft fork is an additional change that turns a +long-existing network relay policy into a consensus rule. The +`OP_CHECKMULTISIG` and `OP_CHECKMULTISIGVERIFY` opcodes consume an extra +stack element ("dummy element") after signature validation. The dummy +element is not inspected in any manner, and could be replaced by any +value without invalidating the script. + +Because any value can be used for this dummy element, it's possible for +a third-party to insert data into other people's transactions, changing +the transaction's txid (called transaction malleability) and possibly +causing other problems. + +Since Bitcoin Core 0.10.0, nodes have defaulted to only relaying and +mining transactions whose dummy element was a null value (0x00, also +called OP_0). The null dummy soft fork turns this relay rule into a +consensus rule both for non-segwit transactions and segwit transactions, +so that this method of mutating transactions is permanently eliminated +from the network. + +Signaling for the null dummy soft fork is done by signaling support +for segwit, and the null dummy soft fork will activate at the same time +as segwit. + +For more information, please see [BIP147][]. + +[BIP147]: https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki + +Low-level RPC changes +--------------------- + +- `importprunedfunds` only accepts two required arguments. Some versions accept + an optional third arg, which was always ignored. Make sure to never pass more + than two arguments. + + +Linux ARM builds +---------------- + +With the 0.13.0 release, pre-built Linux ARM binaries were added to the set of +uploaded executables. Additional detail on the ARM architecture targeted by each +is provided below. + +The following extra files can be found in the download directory or torrent: + +- `bitcoin-${VERSION}-arm-linux-gnueabihf.tar.gz`: Linux binaries targeting + the 32-bit ARMv7-A architecture. +- `bitcoin-${VERSION}-aarch64-linux-gnu.tar.gz`: Linux binaries targeting + the 64-bit ARMv8-A architecture. + +ARM builds are still experimental. If you have problems on a certain device or +Linux distribution combination please report them on the bug tracker, it may be +possible to resolve them. Note that the device you use must be (backward) +compatible with the architecture targeted by the binary that you use. +For example, a Raspberry Pi 2 Model B or Raspberry Pi 3 Model B (in its 32-bit +execution state) device, can run the 32-bit ARMv7-A targeted binary. However, +no model of Raspberry Pi 1 device can run either binary because they are all +ARMv6 architecture devices that are not compatible with ARMv7-A or ARMv8-A. + +Note that Android is not considered ARM Linux in this context. The executables +are not expected to work out of the box on Android. + + +0.13.1 Change log +================= + +Detailed release notes follow. This overview includes changes that affect +behavior, not code moves, refactors and string updates. For convenience in locating +the code changes and accompanying discussion, both the pull request and +git merge commit are mentioned. + +### Consensus +- #8636 `9dfa0c8` Implement NULLDUMMY softfork (BIP147) (jl2012) +- #8848 `7a34a46` Add NULLDUMMY verify flag in bitcoinconsensus.h (jl2012) +- #8937 `8b66659` Define start and end time for segwit deployment (sipa) + +### RPC and other APIs +- #8581 `526d2b0` Drop misleading option in importprunedfunds (MarcoFalke) +- #8699 `a5ec248` Remove createwitnessaddress RPC command (jl2012) +- #8780 `794b007` Deprecate getinfo (MarcoFalke) +- #8832 `83ad563` Throw JSONRPCError when utxo set can not be read (MarcoFalke) +- #8884 `b987348` getblockchaininfo help: pruneheight is the lowest, not highest, block (luke-jr) +- #8858 `3f508ed` rpc: Generate auth cookie in hex instead of base64 (laanwj) +- #8951 `7c2bf4b` RPC/Mining: getblocktemplate: Update and fix formatting of help (luke-jr) + +### Block and transaction handling +- #8611 `a9429ca` Reduce default number of blocks to check at startup (sipa) +- #8634 `3e80ab7` Add policy: null signature for failed CHECK(MULTI)SIG (jl2012) +- #8525 `1672225` Do not store witness txn in rejection cache (sipa) +- #8499 `9777fe1` Add several policy limits and disable uncompressed keys for segwit scripts (jl2012) +- #8526 `0027672` Make non-minimal OP_IF/NOTIF argument non-standard for P2WSH (jl2012) +- #8524 `b8c79a0` Precompute sighashes (sipa) +- #8651 `b8c79a0` Predeclare PrecomputedTransactionData as struct (sipa) + +### P2P protocol and network code +- #8740 `42ea51a` No longer send local address in addrMe (laanwj) +- #8427 `69d1cd2` Ignore `notfound` P2P messages (laanwj) +- #8573 `4f84082` Set jonasschnellis dns-seeder filter flag (jonasschnelli) +- #8712 `23feab1` Remove maxuploadtargets recommended minimum (jonasschnelli) +- #8862 `7ae6242` Fix a few cases where messages were sent after requested disconnect (theuni) +- #8393 `fe1975a` Support for compact blocks together with segwit (sipa) +- #8282 `2611ad7` Feeler connections to increase online addrs in the tried table (EthanHeilman) +- #8612 `2215c22` Check for compatibility with download in FindNextBlocksToDownload (sipa) +- #8606 `bbf379b` Fix some locks (sipa) +- #8594 `ab295bb` Do not add random inbound peers to addrman (gmaxwell) +- #8940 `5b4192b` Add x9 service bit support to dnsseed.bluematt.me, seed.bitcoinstats.com (TheBlueMatt, cdecker) +- #8944 `685e4c7` Remove bogus assert on number of oubound connections. (TheBlueMatt) +- #8949 `0dbc48a` Be more agressive in getting connections to peers with relevant services (gmaxwell) + +### Build system +- #8293 `fa5b249` Allow building libbitcoinconsensus without any univalue (luke-jr) +- #8492 `8b0bdd3` Allow building bench_bitcoin by itself (luke-jr) +- #8563 `147003c` Add configure check for -latomic (ajtowns) +- #8626 `ea51b0f` Berkeley DB v6 compatibility fix (netsafe) +- #8520 `75f2065` Remove check for `openssl/ec.h` (laanwj) + +### GUI +- #8481 `d9f0d4e` Fix minimize and close bugs (adlawren) +- #8487 `a37cec5` Persist the datadir after option reset (achow101) +- #8697 `41fd852` Fix op order to append first alert (rodasmith) +- #8678 `8e03382` Fix UI bug that could result in paying unexpected fee (jonasschnelli) +- #8911 `7634d8e` Translate all files, even if wallet disabled (laanwj) +- #8540 `1db3352` Fix random segfault when closing "Choose data directory" dialog (laanwj) +- #7579 `f1c0d78` Show network/chain errors in the GUI (jonasschnelli) + +### Wallet +- #8443 `464dedd` Trivial cleanup of HD wallet changes (jonasschnelli) +- #8539 `cb07f19` CDB: fix debug output (crowning-) +- #8664 `091cdeb` Fix segwit-related wallet bug (sdaftuar) +- #8693 `c6a6291` Add witness address to address book (instagibbs) +- #8765 `6288659` Remove "unused" ThreadFlushWalletDB from removeprunedfunds (jonasschnelli) + +### Tests and QA +- #8713 `ae8c7df` create_cache: Delete temp dir when done (MarcoFalke) +- #8716 `e34374e` Check legacy wallet as well (MarcoFalke) +- #8750 `d6ebe13` Refactor RPCTestHandler to prevent TimeoutExpired (MarcoFalke) +- #8652 `63462c2` remove root test directory for RPC tests (yurizhykin) +- #8724 `da94272` walletbackup: Sync blocks inside the loop (MarcoFalke) +- #8400 `bea02dc` enable rpcbind_test (yurizhykin) +- #8417 `f70be14` Add walletdump RPC test (including HD- & encryption-tests) (jonasschnelli) +- #8419 `a7aa3cc` Enable size accounting in mining unit tests (sdaftuar) +- #8442 `8bb1efd` Rework hd wallet dump test (MarcoFalke) +- #8528 `3606b6b` Update p2p-segwit.py to reflect correct behavior (instagibbs) +- #8531 `a27cdd8` abandonconflict: Use assert_equal (MarcoFalke) +- #8667 `6b07362` Fix SIGHASH_SINGLE bug in test_framework SignatureHash (jl2012) +- #8673 `03b0196` Fix obvious assignment/equality error in test (JeremyRubin) +- #8739 `cef633c` Fix broken sendcmpct test in p2p-compactblocks.py (sdaftuar) +- #8418 `ff893aa` Add tests for compact blocks (sdaftuar) +- #8803 `375437c` Ping regularly in p2p-segwit.py to keep connection alive (jl2012) +- #8827 `9bbe66e` Split up slow RPC calls to avoid pruning test timeouts (sdaftuar) +- #8829 `2a8bca4` Add bitcoin-tx JSON tests (jnewbery) +- #8834 `1dd1783` blockstore: Switch to dumb dbm (MarcoFalke) +- #8835 `d87227d` nulldummy.py: Don't run unused code (MarcoFalke) +- #8836 `eb18cc1` bitcoin-util-test.py should fail if the output file is empty (jnewbery) +- #8839 `31ab2f8` Avoid ConnectionResetErrors during RPC tests (laanwj) +- #8840 `cbc3fe5` Explicitly set encoding to utf8 when opening text files (laanwj) +- #8841 `3e4abb5` Fix nulldummy test (jl2012) +- #8854 `624a007` Fix race condition in p2p-compactblocks test (sdaftuar) +- #8857 `1f60d45` mininode: Only allow named args in wait_until (MarcoFalke) +- #8860 `0bee740` util: Move wait_bitcoinds() into stop_nodes() (MarcoFalke) +- #8882 `b73f065` Fix race conditions in p2p-compactblocks.py and sendheaders.py (sdaftuar) +- #8904 `cc6f551` Fix compact block shortids for a test case (dagurval) + +### Documentation +- #8754 `0e2c6bd` Target protobuf 2.6 in OS X build notes. (fanquake) +- #8461 `b17a3f9` Document return value of networkhashps for getmininginfo RPC endpoint (jlopp) +- #8512 `156e305` Corrected JSON typo on setban of net.cpp (sevastos) +- #8683 `8a7d7ff` Fix incorrect file name bitcoin.qrc (bitcoinsSG) +- #8891 `5e0dd9e` Update bips.md for Segregated Witness (fanquake) +- #8545 `863ae74` Update git-subtree-check.sh README (MarcoFalke) +- #8607 `486650a` Fix doxygen off-by-one comments, fix typos (MarcoFalke) +- #8560 `c493f43` Fix two VarInt examples in serialize.h (cbarcenas) +- #8737 `084cae9` UndoReadFromDisk works on undo files (rev), not on block files (paveljanik) +- #8625 `0a35573` Clarify statement about parallel jobs in rpc-tests.py (isle2983) +- #8624 `0e6d753` build: Mention curl (MarcoFalke) +- #8604 `b09e13c` build,doc: Update for 0.13.0+ and OpenBSD 5.9 (laanwj) +- #8939 `06d15fb` Update implemented bips for 0.13.1 (sipa) + +### Miscellaneous +- #8742 `d31ac72` Specify Protobuf version 2 in paymentrequest.proto (fanquake) +- #8414,#8558,#8676,#8700,#8701,#8702 Add missing copyright headers (isle2983, kazcw) +- #8899 `4ed2627` Fix wake from sleep issue with Boost 1.59.0 (fanquake) +- #8817 `bcf3806` update bitcoin-tx to output witness data (jnewbery) +- #8513 `4e5fc31` Fix a type error that would not compile on OSX. (JeremyRubin) +- #8392 `30eac2d` Fix several node initialization issues (sipa) +- #8548 `305d8ac` Use `__func__` to get function name for output printing (MarcoFalke) +- #8291 `a987431` [util] CopyrightHolders: Check for untranslated substitution (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- adlawren +- Alexey Vesnin +- Anders Øyvind Urke-Sætre +- Andrew Chow +- Anthony Towns +- BtcDrak +- Chris Stewart +- Christian Barcenas +- Christian Decker +- Cory Fields +- crowning- +- Dagur Valberg Johannsson +- David A. Harding +- Eric Lombrozo +- Ethan Heilman +- fanquake +- Gaurav Rana +- Gregory Maxwell +- instagibbs +- isle2983 +- Jameson Lopp +- Jeremy Rubin +- jnewbery +- Johnson Lau +- Jonas Schnelli +- jonnynewbs +- Justin Camarena +- Kaz Wesley +- leijurv +- Luke Dashjr +- MarcoFalke +- Marty Jones +- Matt Corallo +- Micha +- Michael Ford +- mruddy +- Pavel JanÃk +- Pieter Wuille +- rodasmith +- Sev +- Suhas Daftuar +- whythat +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 63f75fb399..61f05b0771 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -12,6 +12,7 @@ Before every minor and major release: * Update [bips.md](bips.md) to account for changes since the last release. * Update version in sources (see below) * Write release notes (see below) +* Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. Before every major release: @@ -279,6 +280,8 @@ bitcoin.org (see below for bitcoin.org update instructions). - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) - - Add release notes for the new version to the directory `doc/release-notes` in git master + - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) + + - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. - Celebrate diff --git a/doc/unit-tests.md b/doc/unit-tests.md deleted file mode 100644 index afaece829c..0000000000 --- a/doc/unit-tests.md +++ /dev/null @@ -1,18 +0,0 @@ -Compiling/running unit tests ------------------------------------- - -Unit tests will be automatically compiled if dependencies were met in `./configure` -and tests weren't explicitly disabled. - -After configuring, they can be run with `make check`. - -To run the bitcoind tests manually, launch `src/test/test_bitcoin`. - -To add more bitcoind tests, add `BOOST_AUTO_TEST_CASE` functions to the existing -.cpp files in the `test/` directory or add new .cpp files that -implement new BOOST_AUTO_TEST_SUITE sections. - -To run the bitcoin-qt tests manually, launch `src/qt/test/test_bitcoin-qt` - -To add more bitcoin-qt tests, add them to the `src/qt/test/` directory and -the `src/qt/test/test_main.cpp` file. diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 778f8d8a77..58bd00fdfc 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -246,6 +246,10 @@ class RPCTestHandler: self.test_list = test_list self.flags = flags self.num_running = 0 + # In case there is a graveyard of zombie bitcoinds, we can apply a + # pseudorandom offset to hopefully jump over them. + # (625 is PORT_RANGE/MAX_NODES) + self.portseed_offset = int(time.time() * 1000) % 625 self.jobs = [] def get_next(self): @@ -253,7 +257,7 @@ class RPCTestHandler: # Add tests self.num_running += 1 t = self.test_list.pop(0) - port_seed = ["--portseed=%s" % len(self.test_list)] + port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) self.jobs.append((t, diff --git a/qa/rpc-tests/multi_rpc.py b/qa/rpc-tests/multi_rpc.py index cdeb94caa0..95d9090ce2 100755 --- a/qa/rpc-tests/multi_rpc.py +++ b/qa/rpc-tests/multi_rpc.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -# Test mulitple rpc user config option rpcauth +# Test multiple rpc user config option rpcauth # from test_framework.test_framework import BitcoinTestFramework diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 6b5d477131..ab4b809ded 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -27,6 +27,7 @@ class TestNode(SingleNodeConnCB): self.last_cmpctblock = None self.block_announced = False self.last_getdata = None + self.last_getheaders = None self.last_getblocktxn = None self.last_block = None self.last_blocktxn = None @@ -64,6 +65,9 @@ class TestNode(SingleNodeConnCB): def on_getdata(self, conn, message): self.last_getdata = message + def on_getheaders(self, conn, message): + self.last_getheaders = message + def on_getblocktxn(self, conn, message): self.last_getblocktxn = message @@ -186,12 +190,15 @@ class CompactBlocksTest(BitcoinTestFramework): def check_announcement_of_new_block(node, peer, predicate): peer.clear_block_announcement() - node.generate(1) - got_message = wait_until(lambda: peer.block_announced, timeout=30) + block_hash = int(node.generate(1)[0], 16) + peer.wait_for_block_announcement(block_hash, timeout=30) assert(peer.block_announced) assert(got_message) + with mininode_lock: - assert(predicate(peer)) + assert predicate(peer), ( + "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( + block_hash, peer.last_cmpctblock, peer.last_inv)) # We shouldn't get any block announcements via cmpctblock yet. check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) @@ -300,8 +307,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert(segwit_tx_generated) # check that our test is not broken # Wait until we've seen the block announcement for the resulting tip - tip = int(self.nodes[0].getbestblockhash(), 16) - assert(self.test_node.wait_for_block_announcement(tip)) + tip = int(node.getbestblockhash(), 16) + assert(test_node.wait_for_block_announcement(tip)) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -393,6 +400,9 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) + success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30) + assert(success) + test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) @@ -589,8 +599,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.sha256) def test_getblocktxn_handler(self, node, test_node, version): - # bitcoind won't respond for blocks whose height is more than 15 blocks - # deep. + # bitcoind will not send blocktxn responses for blocks whose height is + # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 chain_height = node.getblockcount() current_height = chain_height @@ -623,11 +633,17 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.last_blocktxn = None current_height -= 1 - # Next request should be ignored, as we're past the allowed depth. + # Next request should send a full block response, as we're past the + # allowed depth for a blocktxn response. block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + with mininode_lock: + test_node.last_block = None + test_node.last_blocktxn = None test_node.send_and_ping(msg) with mininode_lock: + test_node.last_block.block.calc_sha256() + assert_equal(test_node.last_block.block.sha256, int(block_hash, 16)) assert_equal(test_node.last_blocktxn, None) def test_compactblocks_not_at_tip(self, node, test_node): @@ -648,6 +664,8 @@ class CompactBlocksTest(BitcoinTestFramework): node.generate(1) wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() + with mininode_lock: + test_node.last_block = None test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) success = wait_until(lambda: test_node.last_block is not None, timeout=30) assert(success) @@ -708,6 +726,33 @@ class CompactBlocksTest(BitcoinTestFramework): l.last_cmpctblock.header_and_shortids.header.calc_sha256() assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + # Test that we don't get disconnected if we relay a compact block with valid header, + # but invalid transactions. + def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): + assert(len(self.utxos)) + utxo = self.utxos[0] + + block = self.build_block_with_transactions(node, utxo, 5) + del block.vtx[3] + block.hashMerkleRoot = block.calc_merkle_root() + if use_segwit: + # If we're testing with segwit, also drop the coinbase witness, + # but include the witness commitment. + add_witness_commitment(block) + block.vtx[0].wit.vtxinwit = [] + block.solve() + + # Now send the compact block with all transactions prefilled, and + # verify that we don't get disconnected. + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) + msg = msg_cmpctblock(comp_block.to_p2p()) + test_node.send_and_ping(msg) + + # Check that the tip didn't advance + assert(int(node.getbestblockhash(), 16) is not block.sha256) + test_node.sync_with_ping() + # Helper for enabling cb announcements # Send the sendcmpct request and sync headers def request_cb_announcements(self, peer, node, version): @@ -798,6 +843,11 @@ class CompactBlocksTest(BitcoinTestFramework): self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + print("\tTesting handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) + # Advance to segwit activation print ("\nAdvancing to segwit activation\n") self.activate_segwit(self.nodes[1]) @@ -844,6 +894,11 @@ class CompactBlocksTest(BitcoinTestFramework): self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + print("\tTesting handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + print("\tTesting invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/qa/rpc-tests/p2p-fullblocktest.py b/qa/rpc-tests/p2p-fullblocktest.py index 9aee81164f..e4b889d761 100755 --- a/qa/rpc-tests/p2p-fullblocktest.py +++ b/qa/rpc-tests/p2p-fullblocktest.py @@ -351,7 +351,7 @@ class FullBlockTest(ComparisonTestFramework): block(22, spend=out[5]) yield rejected() - # Create a block on either side of MAX_BLOCK_SIZE and make sure its accepted/rejected + # Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) # \-> b24 (6) -> b25 (7) @@ -359,24 +359,24 @@ class FullBlockTest(ComparisonTestFramework): tip(15) b23 = block(23, spend=out[6]) tx = CTransaction() - script_length = MAX_BLOCK_SIZE - len(b23.serialize()) - 69 + script_length = MAX_BLOCK_BASE_SIZE - len(b23.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 0))) b23 = update_block(23, [tx]) # Make sure the math above worked out to produce a max-sized block - assert_equal(len(b23.serialize()), MAX_BLOCK_SIZE) + assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE) yield accepted() save_spendable_output() # Make the next block one byte bigger and check that it fails tip(15) b24 = block(24, spend=out[6]) - script_length = MAX_BLOCK_SIZE - len(b24.serialize()) - 69 + script_length = MAX_BLOCK_BASE_SIZE - len(b24.serialize()) - 69 script_output = CScript([b'\x00' * (script_length+1)]) tx.vout = [CTxOut(0, script_output)] b24 = update_block(24, [tx]) - assert_equal(len(b24.serialize()), MAX_BLOCK_SIZE+1) + assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE+1) yield rejected(RejectResult(16, b'bad-blk-length')) block(25, spend=out[7]) @@ -523,12 +523,12 @@ class FullBlockTest(ComparisonTestFramework): tx_new = None tx_last = tx total_size=len(b39.serialize()) - while(total_size < MAX_BLOCK_SIZE): + while(total_size < MAX_BLOCK_BASE_SIZE): tx_new = create_tx(tx_last, 1, 1, p2sh_script) tx_new.vout.append(CTxOut(tx_last.vout[1].nValue - 1, CScript([OP_TRUE]))) tx_new.rehash() total_size += len(tx_new.serialize()) - if total_size >= MAX_BLOCK_SIZE: + if total_size >= MAX_BLOCK_BASE_SIZE: break b39.vtx.append(tx_new) # add tx to block tx_last = tx_new @@ -877,7 +877,7 @@ class FullBlockTest(ComparisonTestFramework): # This checks that a block with a bloated VARINT between the block_header and the array of tx such that - # the block is > MAX_BLOCK_SIZE with the bloated varint, but <= MAX_BLOCK_SIZE without the bloated varint, + # the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint, # does not cause a subsequent, identical block with canonical encoding to be rejected. The test does not # care whether the bloated block is accepted or rejected; it only cares that the second block is accepted. # @@ -901,12 +901,12 @@ class FullBlockTest(ComparisonTestFramework): tx = CTransaction() # use canonical serialization to calculate size - script_length = MAX_BLOCK_SIZE - len(b64a.normal_serialize()) - 69 + script_length = MAX_BLOCK_BASE_SIZE - len(b64a.normal_serialize()) - 69 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) b64a = update_block("64a", [tx]) - assert_equal(len(b64a.serialize()), MAX_BLOCK_SIZE + 8) + assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) yield TestInstance([[self.tip, None]]) # comptool workaround: to make sure b64 is delivered, manually erase b64a from blockstore @@ -916,7 +916,7 @@ class FullBlockTest(ComparisonTestFramework): b64 = CBlock(b64a) b64.vtx = copy.deepcopy(b64a.vtx) assert_equal(b64.hash, b64a.hash) - assert_equal(len(b64.serialize()), MAX_BLOCK_SIZE) + assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE) self.blocks[64] = b64 update_block(64, []) yield accepted() @@ -1250,12 +1250,12 @@ class FullBlockTest(ComparisonTestFramework): for i in range(89, LARGE_REORG_SIZE + 89): b = block(i, spend) tx = CTransaction() - script_length = MAX_BLOCK_SIZE - len(b.serialize()) - 69 + script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0))) b = update_block(i, [tx]) - assert_equal(len(b.serialize()), MAX_BLOCK_SIZE) + assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE) test1.blocks_and_transactions.append([self.tip, True]) save_spendable_output() spend = get_spendable_output() diff --git a/qa/rpc-tests/p2p-segwit.py b/qa/rpc-tests/p2p-segwit.py index 09ab1b80fc..51adec5cae 100755 --- a/qa/rpc-tests/p2p-segwit.py +++ b/qa/rpc-tests/p2p-segwit.py @@ -64,6 +64,9 @@ class TestNode(NodeConnCB): self.getdataset.add(inv.hash) self.last_getdata = message + def on_getheaders(self, conn, message): + self.last_getheaders = message + def on_pong(self, conn, message): self.last_pong = message @@ -97,6 +100,10 @@ class TestNode(NodeConnCB): test_function = lambda: self.last_getdata != None self.sync(test_function, timeout) + def wait_for_getheaders(self, timeout=60): + test_function = lambda: self.last_getheaders != None + self.sync(test_function, timeout) + def wait_for_inv(self, expected_inv, timeout=60): test_function = lambda: self.last_inv != expected_inv self.sync(test_function, timeout) @@ -111,12 +118,15 @@ class TestNode(NodeConnCB): def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with mininode_lock: self.last_getdata = None + self.last_getheaders = None + msg = msg_headers() + msg.headers = [ CBlockHeader(block) ] if use_header: - msg = msg_headers() - msg.headers = [ CBlockHeader(block) ] self.send_message(msg) else: self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) + self.wait_for_getheaders() + self.send_message(msg) self.wait_for_getdata() return @@ -185,11 +195,6 @@ class SegWitTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 - def add_options(self, parser): - parser.add_option("--oldbinary", dest="oldbinary", - default=None, - help="pre-segwit bitcoind binary for upgrade testing") - def setup_network(self): self.nodes = [] self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-logtimemicros=1", "-whitelist=127.0.0.1"])) @@ -197,12 +202,9 @@ class SegWitTest(BitcoinTestFramework): self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-logtimemicros=1", "-whitelist=127.0.0.1", "-acceptnonstdtxn=0"])) connect_nodes(self.nodes[0], 1) - # If an old bitcoind is given, do the upgrade-after-activation test. - self.test_upgrade = False - if (self.options.oldbinary != None): - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1"], binary=self.options.oldbinary)) - connect_nodes(self.nodes[0], 2) - self.test_upgrade = True + # Disable segwit's bip9 parameter to simulate upgrading after activation. + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1", "-bip9params=segwit:0:0"])) + connect_nodes(self.nodes[0], 2) ''' Helpers ''' # Build a block on top of node0's tip. @@ -488,7 +490,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a'*5000000) - assert(get_virtual_size(block) > MAX_BLOCK_SIZE) + assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed @@ -497,7 +499,7 @@ class SegWitTest(BitcoinTestFramework): assert(self.nodes[0].getbestblockhash() != block.hash) block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_SIZE) + assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) assert(self.nodes[0].getbestblockhash() == block.hash) @@ -562,10 +564,10 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [parent_tx, child_tx]) vsize = get_virtual_size(block) - additional_bytes = (MAX_BLOCK_SIZE - vsize)*4 + additional_bytes = (MAX_BLOCK_BASE_SIZE - vsize)*4 i = 0 while additional_bytes > 0: - # Add some more bytes to each input until we hit MAX_BLOCK_SIZE+1 + # Add some more bytes to each input until we hit MAX_BLOCK_BASE_SIZE+1 extra_bytes = min(additional_bytes+1, 55) block.vtx[-1].wit.vtxinwit[int(i/(2*NUM_DROPS))].scriptWitness.stack[i%(2*NUM_DROPS)] = b'a'*(195+extra_bytes) additional_bytes -= extra_bytes @@ -575,7 +577,7 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block) block.solve() vsize = get_virtual_size(block) - assert_equal(vsize, MAX_BLOCK_SIZE + 1) + assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) # Make sure that our test case would exceed the old max-network-message # limit assert(len(block.serialize(True)) > 2*1024*1024) @@ -588,7 +590,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_SIZE) + assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) self.test_node.test_witness_block(block, accepted=True) @@ -1167,7 +1169,7 @@ class SegWitTest(BitcoinTestFramework): if segwit_activated: # tx and tx2 were both accepted. Don't bother trying to reclaim the # P2PKH output; just send tx's first output back to an anyone-can-spend. - sync_mempools(self.nodes) + sync_mempools([self.nodes[0], self.nodes[1]]) tx3.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] tx3.vout = [CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))] tx3.wit.vtxinwit.append(CTxInWitness()) @@ -1423,7 +1425,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx.append(tx) # Test the block periodically, if we're close to maxblocksize - if (get_virtual_size(block) > MAX_BLOCK_SIZE - 1000): + if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): self.update_witness_block_with_transactions(block, []) self.test_node.test_witness_block(block, accepted=True) block = self.build_next_block() @@ -1696,19 +1698,53 @@ class SegWitTest(BitcoinTestFramework): def test_getblocktemplate_before_lockin(self): print("\tTesting getblocktemplate setting of segwit versionbit (before lockin)") - block_version = (self.nodes[0].getblocktemplate())['version'] - assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) + # Node0 is segwit aware, node2 is not. + for node in [self.nodes[0], self.nodes[2]]: + gbt_results = node.getblocktemplate() + block_version = gbt_results['version'] + # If we're not indicating segwit support, we should not be signalling + # for segwit activation, nor should we get a witness commitment. + assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) + assert('default_witness_commitment' not in gbt_results) # Workaround: # Can either change the tip, or change the mempool and wait 5 seconds # to trigger a recomputation of getblocktemplate. - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) # Using mocktime lets us avoid sleep() + sync_mempools(self.nodes) self.nodes[0].setmocktime(int(time.time())+10) - - block_version = self.nodes[0].getblocktemplate({"rules" : ["segwit"]})['version'] - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - self.nodes[0].setmocktime(0) # undo mocktime + self.nodes[2].setmocktime(int(time.time())+10) + + for node in [self.nodes[0], self.nodes[2]]: + gbt_results = node.getblocktemplate({"rules" : ["segwit"]}) + block_version = gbt_results['version'] + if node == self.nodes[2]: + # If this is a non-segwit node, we should still not get a witness + # commitment, nor a version bit signalling segwit. + assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) + assert('default_witness_commitment' not in gbt_results) + else: + # For segwit-aware nodes, check the version bit and the witness + # commitment are correct. + assert(block_version & (1 << VB_WITNESS_BIT) != 0) + assert('default_witness_commitment' in gbt_results) + witness_commitment = gbt_results['default_witness_commitment'] + + # TODO: this duplicates some code from blocktools.py, would be nice + # to refactor. + # Check that default_witness_commitment is present. + block = CBlock() + witness_root = block.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) + check_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(0))) + from test_framework.blocktools import WITNESS_COMMITMENT_HEADER + output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(check_commitment) + script = CScript([OP_RETURN, output_data]) + assert_equal(witness_commitment, bytes_to_hex_str(script)) + + # undo mocktime + self.nodes[0].setmocktime(0) + self.nodes[2].setmocktime(0) # Uncompressed pubkeys are no longer supported in default relay policy, # but (for now) are still valid in blocks. @@ -1948,6 +1984,7 @@ class SegWitTest(BitcoinTestFramework): # Advance to segwit being 'started' self.advance_to_segwit_started() + sync_blocks(self.nodes) self.test_getblocktemplate_before_lockin() sync_blocks(self.nodes) @@ -1990,10 +2027,7 @@ class SegWitTest(BitcoinTestFramework): self.test_signature_version_1() self.test_non_standard_witness() sync_blocks(self.nodes) - if self.test_upgrade: - self.test_upgrade_after_activation(self.nodes[2], 2) - else: - print("\tSkipping upgrade-after-activation test (use --oldbinary to enable)") + self.test_upgrade_after_activation(self.nodes[2], 2) self.test_witness_sigops() diff --git a/qa/rpc-tests/preciousblock.py b/qa/rpc-tests/preciousblock.py index 854dcc7251..3cefa51c0a 100755 --- a/qa/rpc-tests/preciousblock.py +++ b/qa/rpc-tests/preciousblock.py @@ -8,7 +8,12 @@ # from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * +from test_framework.util import ( + assert_equal, + connect_nodes_bi, + sync_chain, + sync_blocks, +) def unidirectional_node_sync_via_rpc(node_src, node_dest): blocks_to_copy = [] @@ -33,84 +38,82 @@ def node_sync_via_rpc(nodes): unidirectional_node_sync_via_rpc(node_src, node_dest) class PreciousTest(BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 3) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + self.extra_args = [["-debug"]] * self.num_nodes def setup_network(self): - self.nodes = [] - self.is_network_split = False - self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-debug"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes = self.setup_nodes() def run_test(self): print("Ensure submitblock can in principle reorg to a competing chain") self.nodes[0].generate(1) - assert(self.nodes[0].getblockcount() == 1) + assert_equal(self.nodes[0].getblockcount(), 1) (hashY, hashZ) = self.nodes[1].generate(2) - assert(self.nodes[1].getblockcount() == 2) + assert_equal(self.nodes[1].getblockcount(), 2) node_sync_via_rpc(self.nodes[0:3]) - assert(self.nodes[0].getbestblockhash() == hashZ) + assert_equal(self.nodes[0].getbestblockhash(), hashZ) print("Mine blocks A-B-C on Node 0") (hashA, hashB, hashC) = self.nodes[0].generate(3) - assert(self.nodes[0].getblockcount() == 5) + assert_equal(self.nodes[0].getblockcount(), 5) print("Mine competing blocks E-F-G on Node 1") (hashE, hashF, hashG) = self.nodes[1].generate(3) - assert(self.nodes[1].getblockcount() == 5) + assert_equal(self.nodes[1].getblockcount(), 5) assert(hashC != hashG) print("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) connect_nodes_bi(self.nodes,0,1) - assert(self.nodes[0].getbestblockhash() == hashC) - assert(self.nodes[1].getbestblockhash() == hashG) + assert_equal(self.nodes[0].getbestblockhash(), hashC) + assert_equal(self.nodes[1].getbestblockhash(), hashG) print("Make Node0 prefer block G") self.nodes[0].preciousblock(hashG) - assert(self.nodes[0].getbestblockhash() == hashG) + assert_equal(self.nodes[0].getbestblockhash(), hashG) print("Make Node0 prefer block C again") self.nodes[0].preciousblock(hashC) - assert(self.nodes[0].getbestblockhash() == hashC) + assert_equal(self.nodes[0].getbestblockhash(), hashC) print("Make Node1 prefer block C") self.nodes[1].preciousblock(hashC) sync_chain(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC - assert(self.nodes[1].getbestblockhash() == hashC) + assert_equal(self.nodes[1].getbestblockhash(), hashC) print("Make Node1 prefer block G again") self.nodes[1].preciousblock(hashG) - assert(self.nodes[1].getbestblockhash() == hashG) + assert_equal(self.nodes[1].getbestblockhash(), hashG) print("Make Node0 prefer block G again") self.nodes[0].preciousblock(hashG) - assert(self.nodes[0].getbestblockhash() == hashG) + assert_equal(self.nodes[0].getbestblockhash(), hashG) print("Make Node1 prefer block C again") self.nodes[1].preciousblock(hashC) - assert(self.nodes[1].getbestblockhash() == hashC) + assert_equal(self.nodes[1].getbestblockhash(), hashC) print("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") self.nodes[0].generate(1) - assert(self.nodes[0].getblockcount() == 6) + assert_equal(self.nodes[0].getblockcount(), 6) sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() - assert(self.nodes[1].getbestblockhash() == hashH) + assert_equal(self.nodes[1].getbestblockhash(), hashH) print("Node1 should not be able to prefer block C anymore") self.nodes[1].preciousblock(hashC) - assert(self.nodes[1].getbestblockhash() == hashH) + assert_equal(self.nodes[1].getbestblockhash(), hashH) print("Mine competing blocks I-J-K-L on Node 2") self.nodes[2].generate(4) - assert(self.nodes[2].getblockcount() == 6) + assert_equal(self.nodes[2].getblockcount(), 6) hashL = self.nodes[2].getbestblockhash() print("Connect nodes and check no reorg occurs") node_sync_via_rpc(self.nodes[0:3]) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) - assert(self.nodes[0].getbestblockhash() == hashH) - assert(self.nodes[1].getbestblockhash() == hashH) - assert(self.nodes[2].getbestblockhash() == hashL) + assert_equal(self.nodes[0].getbestblockhash(), hashH) + assert_equal(self.nodes[1].getbestblockhash(), hashH) + assert_equal(self.nodes[2].getbestblockhash(), hashL) print("Make Node1 prefer block L") self.nodes[1].preciousblock(hashL) - assert(self.nodes[1].getbestblockhash() == hashL) + assert_equal(self.nodes[1].getbestblockhash(), hashL) print("Make Node2 prefer block H") self.nodes[2].preciousblock(hashH) - assert(self.nodes[2].getbestblockhash() == hashH) + assert_equal(self.nodes[2].getbestblockhash(), hashH) if __name__ == '__main__': PreciousTest().main() diff --git a/qa/rpc-tests/prioritise_transaction.py b/qa/rpc-tests/prioritise_transaction.py index e1771231c0..85afeab2e3 100755 --- a/qa/rpc-tests/prioritise_transaction.py +++ b/qa/rpc-tests/prioritise_transaction.py @@ -9,7 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import COIN, MAX_BLOCK_SIZE +from test_framework.mininode import COIN, MAX_BLOCK_BASE_SIZE class PrioritiseTransactionTest(BitcoinTestFramework): @@ -42,7 +42,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[start_range:end_range], (i+1)*base_fee) # Make sure that the size of each group of transactions exceeds - # MAX_BLOCK_SIZE -- otherwise the test needs to be revised to create + # MAX_BLOCK_BASE_SIZE -- otherwise the test needs to be revised to create # more transactions. mempool = self.nodes[0].getrawmempool(True) sizes = [0, 0, 0] @@ -50,7 +50,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): for j in txids[i]: assert(j in mempool) sizes[i] += mempool[j]['size'] - assert(sizes[i] > MAX_BLOCK_SIZE) # Fail => raise utxo_count + assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined (lower diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py index 27160cae07..9ccc0ffbb0 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/qa/rpc-tests/proxy_test.py @@ -4,10 +4,16 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import socket +import os from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * +from test_framework.util import ( + PORT_MIN, + PORT_RANGE, + start_nodes, + assert_equal, +) from test_framework.netutil import test_ipv6_local ''' Test plan: @@ -33,6 +39,8 @@ addnode connect to onion addnode connect to generic DNS name ''' +RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports + class ProxyTest(BitcoinTestFramework): def __init__(self): @@ -44,19 +52,19 @@ class ProxyTest(BitcoinTestFramework): # Create two proxies on different ports # ... one unauthenticated self.conf1 = Socks5Configuration() - self.conf1.addr = ('127.0.0.1', 13000 + (os.getpid() % 1000)) + self.conf1.addr = ('127.0.0.1', RANGE_BEGIN + (os.getpid() % 1000)) self.conf1.unauth = True self.conf1.auth = False # ... one supporting authenticated and unauthenticated (Tor) self.conf2 = Socks5Configuration() - self.conf2.addr = ('127.0.0.1', 14000 + (os.getpid() % 1000)) + self.conf2.addr = ('127.0.0.1', RANGE_BEGIN + 1000 + (os.getpid() % 1000)) self.conf2.unauth = True self.conf2.auth = True if self.have_ipv6: # ... one on IPv6 with similar configuration self.conf3 = Socks5Configuration() self.conf3.af = socket.AF_INET6 - self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) + self.conf3.addr = ('::1', RANGE_BEGIN + 2000 + (os.getpid() % 1000)) self.conf3.unauth = True self.conf3.auth = True else: diff --git a/qa/rpc-tests/rawtransactions.py b/qa/rpc-tests/rawtransactions.py index ab6d2e8def..33a6f2b0cc 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/qa/rpc-tests/rawtransactions.py @@ -2,11 +2,15 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""rawtranscation RPCs QA test. -# -# Test re-org scenarios with a mempool that contains transactions -# that spend (directly or indirectly) coinbase transactions. -# +# Tests the following RPCs: +# - createrawtransaction +# - signrawtransaction +# - sendrawtransaction +# - decoderawtransaction +# - getrawtransaction +""" from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * @@ -138,6 +142,33 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + # getrawtransaction tests + # 1. valid parameters - only supply txid + txHash = rawTx["hash"] + assert_equal(self.nodes[0].getrawtransaction(txHash), rawTxSigned['hex']) + + # 2. valid parameters - supply txid and 0 for non-verbose + assert_equal(self.nodes[0].getrawtransaction(txHash, 0), rawTxSigned['hex']) + + # 3. valid parameters - supply txid and False for non-verbose + assert_equal(self.nodes[0].getrawtransaction(txHash, False), rawTxSigned['hex']) + + # 4. valid parameters - supply txid and 1 for verbose. + # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. + assert_equal(self.nodes[0].getrawtransaction(txHash, 1)["hex"], rawTxSigned['hex']) + + # 5. valid parameters - supply txid and True for non-verbose + assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) + + # 6. invalid parameters - supply txid and string "Flase" + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, "Flase") + + # 7. invalid parameters - supply txid and empty array + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, []) + + # 8. invalid parameters - supply txid and empty dict + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, txHash, {}) + inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } rawtx = self.nodes[0].createrawtransaction(inputs, outputs) diff --git a/qa/rpc-tests/sendheaders.py b/qa/rpc-tests/sendheaders.py index 81b2442e6a..37b98c576e 100755 --- a/qa/rpc-tests/sendheaders.py +++ b/qa/rpc-tests/sendheaders.py @@ -348,14 +348,13 @@ class SendHeadersTest(BitcoinTestFramework): if j == 0: # Announce via inv test_node.send_block_inv(tip) - test_node.wait_for_getdata([tip], timeout=5) + test_node.wait_for_getheaders(timeout=5) + # Should have received a getheaders now + test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements - inv_node.send_block_inv(tip) - # Should have received a getheaders as well! - test_node.send_header_for_blocks(blocks) - test_node.wait_for_getdata([x.sha256 for x in blocks[0:-1]], timeout=5) - [ inv_node.send_block_inv(x.sha256) for x in blocks[0:-1] ] + [ inv_node.send_block_inv(x.sha256) for x in blocks ] + test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=5) inv_node.sync_with_ping() else: # Announce via headers diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index d76fba4b07..74a74f679a 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -225,9 +225,9 @@ class EstimateFeeTest(BitcoinTestFramework): self.memutxo, Decimal("0.005"), min_fee, min_fee) tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee)/tx_kbytes) - sync_mempools(self.nodes[0:3],.1) + sync_mempools(self.nodes[0:3], wait=.1) mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"] - sync_blocks(self.nodes[0:3],.1) + sync_blocks(self.nodes[0:3], wait=.1) # update which txouts are confirmed newmem = [] for utx in self.memutxo: @@ -259,7 +259,7 @@ class EstimateFeeTest(BitcoinTestFramework): while len(self.nodes[1].getrawmempool()) > 0: self.nodes[1].generate(1) - sync_blocks(self.nodes[0:3],.1) + sync_blocks(self.nodes[0:3], wait=.1) print("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb, 2) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index fd7f32b5c6..9bee1962e2 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -1,6 +1,6 @@ """ - Copyright 2011 Jeff Garzik + Copyright (c) 2011 Jeff Garzik AuthServiceProxy has the following improvements over python-jsonrpc's ServiceProxy class: @@ -42,6 +42,7 @@ import base64 import decimal import json import logging +import socket try: import urllib.parse as urlparse except ImportError: @@ -161,7 +162,15 @@ class AuthServiceProxy(object): return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): - http_response = self.__conn.getresponse() + try: + http_response = self.__conn.getresponse() + except socket.timeout as e: + raise JSONRPCException({ + 'code': -344, + 'message': '%r RPC took longer than %f seconds. Consider ' + 'using larger timeout for calls that take ' + 'longer to return.' % (self._service_name, + self.__conn.timeout)}) if http_response is None: raise JSONRPCException({ 'code': -342, 'message': 'missing HTTP response from server'}) diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py index 7c92d3f828..17679fc7e1 100755 --- a/qa/rpc-tests/test_framework/comptool.py +++ b/qa/rpc-tests/test_framework/comptool.py @@ -111,6 +111,11 @@ class TestNode(NodeConnCB): m.locator = self.block_store.get_locator(self.bestblockhash) self.conn.send_message(m) + def send_header(self, header): + m = msg_headers() + m.headers.append(header) + self.conn.send_message(m) + # This assumes BIP31 def send_ping(self, nonce): self.pingMap[nonce] = True @@ -345,8 +350,16 @@ class TestManager(object): # Either send inv's to each node and sync, or add # to invqueue for later inv'ing. if (test_instance.sync_every_block): - [ c.cb.send_inv(block) for c in self.connections ] - self.sync_blocks(block.sha256, 1) + # if we expect success, send inv and sync every block + # if we expect failure, just push the block and see what happens. + if outcome == True: + [ c.cb.send_inv(block) for c in self.connections ] + self.sync_blocks(block.sha256, 1) + else: + [ c.send_message(msg_block(block)) for c in self.connections ] + [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + self.wait_for_pings(self.ping_counter) + self.ping_counter += 1 if (not self.check_results(tip, outcome)): raise AssertionError("Test failed at test %d" % test_number) else: @@ -354,6 +367,8 @@ class TestManager(object): elif isinstance(b_or_t, CBlockHeader): block_header = b_or_t self.block_store.add_header(block_header) + [ c.cb.send_header(block_header) for c in self.connections ] + else: # Tx test runner assert(isinstance(b_or_t, CTransaction)) tx = b_or_t diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 4d238c08d9..91daa4ab1f 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -41,9 +41,10 @@ from test_framework.siphash import siphash256 BIP0031_VERSION = 60000 MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) MAX_INV_SZ = 50000 -MAX_BLOCK_SIZE = 1000000 +MAX_BLOCK_BASE_SIZE = 1000000 COIN = 100000000 # 1 btc in satoshis @@ -951,6 +952,7 @@ class msg_version(object): self.nNonce = random.getrandbits(64) self.strSubVer = MY_SUBVERSION self.nStartingHeight = -1 + self.nRelay = MY_RELAY def deserialize(self, f): self.nVersion = struct.unpack("<i", f.read(4))[0] @@ -960,21 +962,32 @@ class msg_version(object): self.nTime = struct.unpack("<q", f.read(8))[0] self.addrTo = CAddress() self.addrTo.deserialize(f) + if self.nVersion >= 106: self.addrFrom = CAddress() self.addrFrom.deserialize(f) self.nNonce = struct.unpack("<Q", f.read(8))[0] self.strSubVer = deser_string(f) - if self.nVersion >= 209: - self.nStartingHeight = struct.unpack("<i", f.read(4))[0] - else: - self.nStartingHeight = None else: self.addrFrom = None self.nNonce = None self.strSubVer = None self.nStartingHeight = None + if self.nVersion >= 209: + self.nStartingHeight = struct.unpack("<i", f.read(4))[0] + else: + self.nStartingHeight = None + + if self.nVersion >= 70001: + # Relay field is optional for version 70001 onwards + try: + self.nRelay = struct.unpack("<b", f.read(1))[0] + except: + self.nRelay = 0 + else: + self.nRelay = 0 + def serialize(self): r = b"" r += struct.pack("<i", self.nVersion) @@ -985,13 +998,14 @@ class msg_version(object): r += struct.pack("<Q", self.nNonce) r += ser_string(self.strSubVer) r += struct.pack("<i", self.nStartingHeight) + r += struct.pack("<b", self.nRelay) return r def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i)' \ + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ % (self.nVersion, self.nServices, time.ctime(self.nTime), repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight) + self.strSubVer, self.nStartingHeight, self.nRelay) class msg_verack(object): diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index e6fc5fd8a2..e6d3e9ab9a 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -139,16 +139,11 @@ class BitcoinTestFramework(object): success = False try: - if not os.path.isdir(self.options.tmpdir): - os.makedirs(self.options.tmpdir) + os.makedirs(self.options.tmpdir, exist_ok=False) self.setup_chain() - self.setup_network() - self.run_test() - success = True - except JSONRPCException as e: print("JSONRPC error: "+e.error['message']) traceback.print_tb(sys.exc_info()[2]) diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index c818af4bd7..99c30f7651 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -121,33 +121,44 @@ def hex_str_to_bytes(hex_str): def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') -def sync_blocks(rpc_connections, wait=1, timeout=60): +def sync_blocks(rpc_connections, *, wait=1, timeout=60): """ - Wait until everybody has the same tip - """ - maxheight = 0 - while timeout > 0: - tips = [ x.waitforblockheight(maxheight, int(wait * 1000)) for x in rpc_connections ] - heights = [ x["height"] for x in tips ] - if tips == [ tips[0] ]*len(tips): - return True - if heights == [ heights[0] ]*len(heights): #heights are the same but hashes are not - raise AssertionError("Block sync failed") - timeout -= wait - maxheight = max(heights) - raise AssertionError("Block sync failed") + Wait until everybody has the same tip. -def sync_chain(rpc_connections, wait=1): + sync_blocks needs to be called with an rpc_connections set that has least + one node already synced to the latest, stable tip, otherwise there's a + chance it might return before all nodes are stably synced. + """ + # Use getblockcount() instead of waitforblockheight() to determine the + # initial max height because the two RPCs look at different internal global + # variables (chainActive vs latestBlock) and the former gets updated + # earlier. + maxheight = max(x.getblockcount() for x in rpc_connections) + start_time = cur_time = time.time() + while cur_time <= start_time + timeout: + tips = [r.waitforblockheight(maxheight, int(wait * 1000)) for r in rpc_connections] + if all(t["height"] == maxheight for t in tips): + if all(t["hash"] == tips[0]["hash"] for t in tips): + return + raise AssertionError("Block sync failed, mismatched block hashes:{}".format( + "".join("\n {!r}".format(tip) for tip in tips))) + cur_time = time.time() + raise AssertionError("Block sync to height {} timed out:{}".format( + maxheight, "".join("\n {!r}".format(tip) for tip in tips))) + +def sync_chain(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same best block """ - while True: - counts = [ x.getbestblockhash() for x in rpc_connections ] - if counts == [ counts[0] ]*len(counts): - break + while timeout > 0: + best_hash = [x.getbestblockhash() for x in rpc_connections] + if best_hash == [best_hash[0]]*len(best_hash): + return time.sleep(wait) + timeout -= wait + raise AssertionError("Chain sync failed: Best block hashes don't match") -def sync_mempools(rpc_connections, wait=1, timeout=60): +def sync_mempools(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same transactions in their memory pools @@ -159,7 +170,7 @@ def sync_mempools(rpc_connections, wait=1, timeout=60): if set(rpc_connections[i].getrawmempool()) == pool: num_match = num_match+1 if num_match == len(rpc_connections): - return True + return time.sleep(wait) timeout -= wait raise AssertionError("Mempool sync failed") @@ -212,7 +223,7 @@ def wait_for_bitcoind_start(process, url, i): raise # unknown IO error except JSONRPCException as e: # Initialization phase if e.error['code'] != -28: # RPC in warmup? - raise # unkown JSON RPC exception + raise # unknown JSON RPC exception time.sleep(0.25) def initialize_chain(test_dir, num_nodes, cachedir): @@ -246,7 +257,7 @@ def initialize_chain(test_dir, num_nodes, cachedir): print("initialize_chain: bitcoind started, waiting for RPC to come up") wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) if os.getenv("PYTHON_DEBUG", ""): - print("initialize_chain: RPC succesfully started") + print("initialize_chain: RPC successfully started") rpcs = [] for i in range(MAX_NODES): @@ -333,7 +344,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= url = rpc_url(i, rpchost) wait_for_bitcoind_start(bitcoind_processes[i], url, i) if os.getenv("PYTHON_DEBUG", ""): - print("start_node: RPC succesfully started") + print("start_node: RPC successfully started") proxy = get_rpc_proxy(url, i, timeout=timewait) if COVERAGE_DIR: @@ -341,7 +352,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= return proxy -def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None): +def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): """ Start multiple bitcoinds, return RPC connections to them """ @@ -350,7 +361,7 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None): rpcs = [] try: for i in range(num_nodes): - rpcs.append(start_node(i, dirname, extra_args[i], rpchost, binary=binary[i])) + rpcs.append(start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) except: # If one node failed to start, stop the others stop_nodes(rpcs) raise @@ -522,10 +533,14 @@ def assert_greater_than(thing1, thing2): raise AssertionError("%s <= %s"%(str(thing1),str(thing2))) def assert_raises(exc, fun, *args, **kwds): + assert_raises_message(exc, None, fun, *args, **kwds) + +def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) - except exc: - pass + except exc as e: + if message is not None and message not in e.error['message']: + raise AssertionError("Expected substring not found:"+e.error['message']) except Exception as e: raise AssertionError("Unexpected exception raised: "+type(e).__name__) else: diff --git a/qa/rpc-tests/wallet-dump.py b/qa/rpc-tests/wallet-dump.py index a37096a40c..c6dc2e3d10 100755 --- a/qa/rpc-tests/wallet-dump.py +++ b/qa/rpc-tests/wallet-dump.py @@ -61,7 +61,11 @@ class WalletDumpTest(BitcoinTestFramework): self.extra_args = [["-keypool=90"]] def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + # Use 1 minute timeout because the initial getnewaddress RPC can take + # longer than the default 30 seconds due to an expensive + # CWallet::TopUpKeyPool call, and the encryptwallet RPC made later in + # the test often takes even longer. + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) def run_test (self): tmpdir = self.options.tmpdir diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index e43f6ea5d2..3c0dc0f4ea 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -71,7 +71,7 @@ class WalletTest (BitcoinTestFramework): unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} self.nodes[2].lockunspent(False, [unspent_0]) - assert_raises(JSONRPCException, self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) + assert_raises_message(JSONRPCException, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 6a34d64cd5..5ca6d9d015 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.7.0</string> + <string>10.8.0</string> <key>LSArchitecturePriority</key> <array> diff --git a/share/rpcuser/rpcuser.py b/share/rpcuser/rpcuser.py index 9fd176908b..f806a810e0 100755 --- a/share/rpcuser/rpcuser.py +++ b/share/rpcuser/rpcuser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +# Copyright (c) 2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/Makefile.am b/src/Makefile.am index ab3104ec63..69ae02f582 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -111,6 +111,7 @@ BITCOIN_CORE_H = \ net.h \ netaddress.h \ netbase.h \ + netmessagemaker.h \ noui.h \ policy/fees.h \ policy/policy.h \ @@ -132,7 +133,7 @@ BITCOIN_CORE_H = \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ - support/pagelocker.h \ + support/lockedpool.h \ sync.h \ threadsafety.h \ timedata.h \ @@ -310,7 +311,7 @@ libbitcoin_common_a_SOURCES = \ libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ - support/pagelocker.cpp \ + support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibc_sanity.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c83432e91a..e58bd9dfbf 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -6,18 +6,27 @@ bin_PROGRAMS += bench/bench_bitcoin BENCH_SRCDIR = bench BENCH_BINARY = bench/bench_bitcoin$(EXEEXT) +RAW_TEST_FILES = \ + bench/data/block413567.raw +GENERATED_TEST_FILES = $(RAW_TEST_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ + bench/checkblock.cpp \ bench/Examples.cpp \ bench/rollingbloom.cpp \ bench/crypto_hash.cpp \ bench/ccoins_caching.cpp \ bench/mempool_eviction.cpp \ bench/verify_script.cpp \ - bench/base58.cpp + bench/base58.cpp \ + bench/lockedpool.cpp \ + bench/perf.cpp \ + bench/perf.h + +nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_TEST_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -44,10 +53,12 @@ endif bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno +CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_TEST_FILES) CLEANFILES += $(CLEAN_BITCOIN_BENCH) +bench/checkblock.cpp: bench/data/block413567.raw.h + bitcoin_bench: $(BENCH_BINARY) bench: $(BENCH_BINARY) FORCE @@ -55,3 +66,12 @@ bench: $(BENCH_BINARY) FORCE bitcoin_bench_clean : FORCE rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY) + +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 1f9a901d75..48abb9b4a2 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -271,6 +271,7 @@ RES_ICONS = \ qt/res/icons/key.png \ qt/res/icons/lock_closed.png \ qt/res/icons/lock_open.png \ + qt/res/icons/network_disabled.png \ qt/res/icons/open.png \ qt/res/icons/overview.png \ qt/res/icons/quit.png \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5ce1bbb896..a14adc7876 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -50,7 +50,6 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ - test/Checkpoints_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -85,6 +84,7 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ + test/test_random.h \ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ @@ -149,16 +149,10 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) - @echo "namespace json_tests{" > $@ - @echo "static unsigned const char $(*F)[] = {" >> $@ - @$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@ - @echo "};};" >> $@ - @echo "Generated $@" - -%.raw.h: %.raw - @$(MKDIR_P) $(@D) - @echo "namespace alert_tests{" > $@ - @echo "static unsigned const char $(*F)[] = {" >> $@ - @$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@ - @echo "};};" >> $@ + @{ \ + echo "namespace json_tests{" && \ + echo "static unsigned const char $(*F)[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};};"; \ + } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index ddf41f92de..23715bbea4 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -103,7 +103,7 @@ bool CBanDB::Read(banmap_t& banSet) if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) return error("%s: Invalid network magic number", __func__); - // de-serialize address data into one CAddrMan object + // de-serialize ban data ssBanlist >> banSet; } catch (const std::exception& e) { diff --git a/src/addrdb.h b/src/addrdb.h index 62835a6fb4..339943ca5a 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -46,7 +46,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(this->nVersion); READWRITE(nCreateTime); READWRITE(nBanUntil); diff --git a/src/addrman.h b/src/addrman.h index e9e137c978..76e7b210f7 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -58,7 +58,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(*(CAddress*)this); READWRITE(source); READWRITE(nLastSuccess); @@ -293,7 +293,7 @@ public: * very little in common. */ template<typename Stream> - void Serialize(Stream &s, int nType, int nVersionDummy) const + void Serialize(Stream &s) const { LOCK(cs); @@ -343,7 +343,7 @@ public: } template<typename Stream> - void Unserialize(Stream& s, int nType, int nVersionDummy) + void Unserialize(Stream& s) { LOCK(cs); @@ -448,11 +448,6 @@ public: Check(); } - unsigned int GetSerializeSize(int nType, int nVersion) const - { - return (CSizeComputer(nType, nVersion) << *this).size(); - } - void Clear() { std::vector<int>().swap(vRandom); @@ -487,6 +482,7 @@ public: //! Return the number of (unique) addresses in all tables. size_t size() const { + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead return vRandom.size(); } @@ -506,13 +502,11 @@ public: //! Add a single address. bool Add(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty = 0) { + LOCK(cs); bool fRet = false; - { - LOCK(cs); - Check(); - fRet |= Add_(addr, source, nTimePenalty); - Check(); - } + Check(); + fRet |= Add_(addr, source, nTimePenalty); + Check(); if (fRet) LogPrint("addrman", "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); return fRet; @@ -521,14 +515,12 @@ public: //! Add multiple addresses. bool Add(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) { + LOCK(cs); int nAdd = 0; - { - LOCK(cs); - Check(); - for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - } + Check(); + for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) + nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; + Check(); if (nAdd) LogPrint("addrman", "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); return nAdd > 0; @@ -537,23 +529,19 @@ public: //! Mark an entry as accessible. void Good(const CService &addr, int64_t nTime = GetAdjustedTime()) { - { - LOCK(cs); - Check(); - Good_(addr, nTime); - Check(); - } + LOCK(cs); + Check(); + Good_(addr, nTime); + Check(); } //! Mark an entry as connection attempted to. void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) { - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, nTime); + Check(); } /** @@ -587,12 +575,10 @@ public: //! Mark an entry as currently-connected-to. void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) { - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } + LOCK(cs); + Check(); + Connected_(addr, nTime); + Check(); } void SetServices(const CService &addr, ServiceFlags nServices) diff --git a/src/amount.h b/src/amount.h index 5e52f37f23..ba0c86040f 100644 --- a/src/amount.h +++ b/src/amount.h @@ -64,7 +64,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nSatoshisPerK); } }; diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 2e61363576..a58ad01b5a 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin developers +// Copyright (c) 2009-2014 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index ba3d620158..5cc52f6e72 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/base58.cpp b/src/base58.cpp index d1d60a6f1d..f7768b5a64 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -25,12 +25,14 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) psz++; // Skip and count leading '1's. int zeroes = 0; + int length = 0; while (*psz == '1') { zeroes++; psz++; } // Allocate enough space in big-endian base256 representation. - std::vector<unsigned char> b256(strlen(psz) * 733 / 1000 + 1); // log(58) / log(256), rounded up. + int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up. + std::vector<unsigned char> b256(size); // Process the characters. while (*psz && !isspace(*psz)) { // Decode base58 character @@ -39,12 +41,14 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) return false; // Apply "b256 = b256 * 58 + ch". int carry = ch - pszBase58; - for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); it != b256.rend(); it++) { + int i = 0; + for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); (carry != 0 || i < length) && (it != b256.rend()); ++it, ++i) { carry += 58 * (*it); *it = carry % 256; carry /= 256; } assert(carry == 0); + length = i; psz++; } // Skip trailing spaces. @@ -53,7 +57,7 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) if (*psz != 0) return false; // Skip leading zeroes in b256. - std::vector<unsigned char>::iterator it = b256.begin(); + std::vector<unsigned char>::iterator it = b256.begin() + (size - length); while (it != b256.end() && *it == 0) it++; // Copy result into output vector. diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 1279c3e7df..a791b5b7fa 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016 the Bitcoin Core developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 227546a7a7..af3d152c9a 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "bench.h" +#include "perf.h" #include <iostream> #include <iomanip> @@ -26,7 +27,9 @@ BenchRunner::BenchRunner(std::string name, BenchFunction func) void BenchRunner::RunAll(double elapsedTimeForOne) { - std::cout << "#Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n"; + perf_init(); + std::cout << "#Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "," + << "min_cycles" << "," << "max_cycles" << "," << "average_cycles" << "\n"; for (std::map<std::string,BenchFunction>::iterator it = benchmarks.begin(); it != benchmarks.end(); ++it) { @@ -35,6 +38,7 @@ BenchRunner::RunAll(double elapsedTimeForOne) BenchFunction& func = it->second; func(state); } + perf_fini(); } bool State::KeepRunning() @@ -44,8 +48,10 @@ bool State::KeepRunning() return true; } double now; + uint64_t nowCycles; if (count == 0) { lastTime = beginTime = now = gettimedouble(); + lastCycles = beginCycles = nowCycles = perf_cpucycles(); } else { now = gettimedouble(); @@ -53,6 +59,13 @@ bool State::KeepRunning() double elapsedOne = elapsed * countMaskInv; if (elapsedOne < minTime) minTime = elapsedOne; if (elapsedOne > maxTime) maxTime = elapsedOne; + + // We only use relative values, so don't have to handle 64-bit wrap-around specially + nowCycles = perf_cpucycles(); + uint64_t elapsedOneCycles = (nowCycles - lastCycles) * countMaskInv; + if (elapsedOneCycles < minCycles) minCycles = elapsedOneCycles; + if (elapsedOneCycles > maxCycles) maxCycles = elapsedOneCycles; + if (elapsed*128 < maxElapsed) { // If the execution was much too fast (1/128th of maxElapsed), increase the count mask by 8x and restart timing. // The restart avoids including the overhead of this code in the measurement. @@ -61,14 +74,20 @@ bool State::KeepRunning() count = 0; minTime = std::numeric_limits<double>::max(); maxTime = std::numeric_limits<double>::min(); + minCycles = std::numeric_limits<uint64_t>::max(); + maxCycles = std::numeric_limits<uint64_t>::min(); return true; } if (elapsed*16 < maxElapsed) { - countMask = ((countMask<<1)|1) & ((1LL<<60)-1); - countMaskInv = 1./(countMask+1); + uint64_t newCountMask = ((countMask<<1)|1) & ((1LL<<60)-1); + if ((count & newCountMask)==0) { + countMask = newCountMask; + countMaskInv = 1./(countMask+1); + } } } lastTime = now; + lastCycles = nowCycles; ++count; if (now - beginTime < maxElapsed) return true; // Keep going @@ -77,7 +96,9 @@ bool State::KeepRunning() // Output results double average = (now-beginTime)/count; - std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n"; + int64_t averageCycles = (nowCycles-beginCycles)/count; + std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << minTime << "," << maxTime << "," << average << "," + << minCycles << "," << maxCycles << "," << averageCycles << "\n"; return false; } diff --git a/src/bench/bench.h b/src/bench/bench.h index f13b145aaf..caf73e949b 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -41,12 +41,18 @@ namespace benchmark { double maxElapsed; double beginTime; double lastTime, minTime, maxTime, countMaskInv; - int64_t count; - int64_t countMask; + uint64_t count; + uint64_t countMask; + uint64_t beginCycles; + uint64_t lastCycles; + uint64_t minCycles; + uint64_t maxCycles; public: State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { minTime = std::numeric_limits<double>::max(); maxTime = std::numeric_limits<double>::min(); + minCycles = std::numeric_limits<uint64_t>::max(); + maxCycles = std::numeric_limits<uint64_t>::min(); countMask = 1; countMaskInv = 1./(countMask + 1); } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp new file mode 100644 index 0000000000..4a564d3fc8 --- /dev/null +++ b/src/bench/checkblock.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "chainparams.h" +#include "main.h" +#include "consensus/validation.h" + +namespace block_bench { +#include "bench/data/block413567.raw.h" +} + +// These are the two major time-sinks which happen after we have fully received +// a block off the wire, but before we can relay the block on to peers using +// compact block relay. + +static void DeserializeBlockTest(benchmark::State& state) +{ + CDataStream stream((const char*)block_bench::block413567, + (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], + SER_NETWORK, PROTOCOL_VERSION); + char a; + stream.write(&a, 1); // Prevent compaction + + while (state.KeepRunning()) { + CBlock block; + stream >> block; + assert(stream.Rewind(sizeof(block_bench::block413567))); + } +} + +static void DeserializeAndCheckBlockTest(benchmark::State& state) +{ + CDataStream stream((const char*)block_bench::block413567, + (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], + SER_NETWORK, PROTOCOL_VERSION); + char a; + stream.write(&a, 1); // Prevent compaction + + Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); + + while (state.KeepRunning()) { + CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here + stream >> block; + assert(stream.Rewind(sizeof(block_bench::block413567))); + + CValidationState validationState; + assert(CheckBlock(block, validationState, params)); + } +} + +BENCHMARK(DeserializeBlockTest); +BENCHMARK(DeserializeAndCheckBlockTest); diff --git a/src/bench/data/block413567.raw b/src/bench/data/block413567.raw Binary files differnew file mode 100644 index 0000000000..67d2d5d382 --- /dev/null +++ b/src/bench/data/block413567.raw diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp new file mode 100644 index 0000000000..5df5b1ac6e --- /dev/null +++ b/src/bench/lockedpool.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "support/lockedpool.h" + +#include <iostream> +#include <vector> + +#define ASIZE 2048 +#define BITER 5000 +#define MSIZE 2048 + +static void LockedPool(benchmark::State& state) +{ + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + + std::vector<void*> addr; + for (int x=0; x<ASIZE; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + while (state.KeepRunning()) { + for (int x=0; x<BITER; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & (MSIZE-1)); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 + } + } + for (void *ptr: addr) + b.free(ptr); + addr.clear(); +} + +BENCHMARK(LockedPool); + diff --git a/src/bench/perf.cpp b/src/bench/perf.cpp new file mode 100644 index 0000000000..1f43e5d3ac --- /dev/null +++ b/src/bench/perf.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "perf.h" + +#if defined(__i386__) || defined(__x86_64__) + +/* These architectures support quering the cycle counter + * from user space, no need for any syscall overhead. + */ +void perf_init(void) { } +void perf_fini(void) { } + +#elif defined(__linux__) + +#include <unistd.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> + +static int fd = -1; +static struct perf_event_attr attr; + +void perf_init(void) +{ + attr.type = PERF_TYPE_HARDWARE; + attr.config = PERF_COUNT_HW_CPU_CYCLES; + fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); +} + +void perf_fini(void) +{ + if (fd != -1) { + close(fd); + } +} + +uint64_t perf_cpucycles(void) +{ + uint64_t result = 0; + if (fd == -1 || read(fd, &result, sizeof(result)) < (ssize_t)sizeof(result)) { + return 0; + } + return result; +} + +#else /* Unhandled platform */ + +void perf_init(void) { } +void perf_fini(void) { } +uint64_t perf_cpucycles(void) { return 0; } + +#endif diff --git a/src/bench/perf.h b/src/bench/perf.h new file mode 100644 index 0000000000..681bd0c8a2 --- /dev/null +++ b/src/bench/perf.h @@ -0,0 +1,37 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +/** Functions for measurement of CPU cycles */ +#ifndef H_PERF +#define H_PERF + +#include <stdint.h> + +#if defined(__i386__) + +static inline uint64_t perf_cpucycles(void) +{ + uint64_t x; + __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); + return x; +} + +#elif defined(__x86_64__) + +static inline uint64_t perf_cpucycles(void) +{ + uint32_t hi, lo; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + return ((uint64_t)lo)|(((uint64_t)hi)<<32); +} +#else + +uint64_t perf_cpucycles(void); + +#endif + +void perf_init(void); +void perf_fini(void); + +#endif // H_PERF diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 2d66448d80..b272640819 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -24,14 +24,13 @@ #include <univalue.h> -using namespace std; - static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; +static const int CONTINUE_EXECUTION=-1; std::string HelpMessageCli() { - string strUsage; + std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); @@ -67,7 +66,11 @@ public: }; -static bool AppInitRPC(int argc, char* argv[]) +// +// This function returns either one of EXIT_ codes when it's expected to stop the process or +// CONTINUE_EXECUTION when it's expected to continue further. +// +static int AppInitRPC(int argc, char* argv[]) { // // Parameters @@ -85,31 +88,35 @@ static bool AppInitRPC(int argc, char* argv[]) } fprintf(stdout, "%s", strUsage.c_str()); - return false; + if (argc < 2) { + fprintf(stderr, "Error: too few parameters\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; } if (!boost::filesystem::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", mapArgs["-datadir"].c_str()); - return false; + return EXIT_FAILURE; } try { ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME), mapArgs, mapMultiArgs); } catch (const std::exception& e) { fprintf(stderr,"Error reading configuration file: %s\n", e.what()); - return false; + return EXIT_FAILURE; } // Check for -testnet or -regtest parameter (BaseParams() calls are only valid after this clause) try { SelectBaseParams(ChainNameFromCommandLine()); } catch (const std::exception& e) { fprintf(stderr, "Error: %s\n", e.what()); - return false; + return EXIT_FAILURE; } if (GetBoolArg("-rpcssl", false)) { fprintf(stderr, "Error: SSL mode for RPC (-rpcssl) is no longer supported.\n"); - return false; + return EXIT_FAILURE; } - return true; + return CONTINUE_EXECUTION; } @@ -178,7 +185,7 @@ static void http_error_cb(enum evhttp_request_error err, void *ctx) } #endif -UniValue CallRPC(const string& strMethod, const UniValue& params) +UniValue CallRPC(const std::string& strMethod, const UniValue& params) { std::string host = GetArg("-rpcconnect", DEFAULT_RPCCONNECT); int port = GetArg("-rpcport", BaseParams().RPCPort()); @@ -186,18 +193,18 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) // Create event base struct event_base *base = event_base_new(); // TODO RAII if (!base) - throw runtime_error("cannot create event_base"); + throw std::runtime_error("cannot create event_base"); // Synchronously look up hostname struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII if (evcon == NULL) - throw runtime_error("create connection failed"); + throw std::runtime_error("create connection failed"); evhttp_connection_set_timeout(evcon, GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); HTTPReply response; struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII if (req == NULL) - throw runtime_error("create http request failed"); + throw std::runtime_error("create http request failed"); #if LIBEVENT_VERSION_NUMBER >= 0x02010300 evhttp_request_set_error_cb(req, http_error_cb); #endif @@ -207,7 +214,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) if (mapArgs["-rpcpassword"] == "") { // Try fall back to cookie-based authentication if no password is provided if (!GetAuthCookie(&strRPCUserColonPass)) { - throw runtime_error(strprintf( + throw std::runtime_error(strprintf( _("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"), GetConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME)).string().c_str())); @@ -240,28 +247,28 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed(strprintf("couldn't connect to server (%d %s)", response.error, http_errorstring(response.error))); + throw CConnectionFailed(strprintf("couldn't connect to server\n(make sure server is running and you are connecting to the correct RPC port: %d %s)", response.error, http_errorstring(response.error))); else if (response.status == HTTP_UNAUTHORIZED) - throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); + throw std::runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) - throw runtime_error(strprintf("server returned HTTP error %d", response.status)); + throw std::runtime_error(strprintf("server returned HTTP error %d", response.status)); else if (response.body.empty()) - throw runtime_error("no response from server"); + throw std::runtime_error("no response from server"); // Parse reply UniValue valReply(UniValue::VSTR); if (!valReply.read(response.body)) - throw runtime_error("couldn't parse reply from server"); + throw std::runtime_error("couldn't parse reply from server"); const UniValue& reply = valReply.get_obj(); if (reply.empty()) - throw runtime_error("expected reply to have result, error and id properties"); + throw std::runtime_error("expected reply to have result, error and id properties"); return reply; } int CommandLineRPC(int argc, char *argv[]) { - string strPrint; + std::string strPrint; int nRet = 0; try { // Skip switches @@ -277,7 +284,7 @@ int CommandLineRPC(int argc, char *argv[]) args.push_back(line); } if (args.size() < 1) - throw runtime_error("too few parameters (need at least command)"); + throw std::runtime_error("too few parameters (need at least command)"); std::string strMethod = args[0]; UniValue params = RPCConvertValues(strMethod, std::vector<std::string>(args.begin()+1, args.end())); @@ -331,7 +338,7 @@ int CommandLineRPC(int argc, char *argv[]) throw; } catch (const std::exception& e) { - strPrint = string("error: ") + e.what(); + strPrint = std::string("error: ") + e.what(); nRet = EXIT_FAILURE; } catch (...) { @@ -354,8 +361,9 @@ int main(int argc, char* argv[]) } try { - if(!AppInitRPC(argc, argv)) - return EXIT_FAILURE; + int ret = AppInitRPC(argc, argv); + if (ret != CONTINUE_EXECUTION) + return ret; } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInitRPC()"); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e09afd632e..346d8e180d 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -26,12 +26,15 @@ #include <boost/algorithm/string.hpp> #include <boost/assign/list_of.hpp> -using namespace std; - static bool fCreateBlank; -static map<string,UniValue> registers; - -static bool AppInitRawTx(int argc, char* argv[]) +static std::map<std::string,UniValue> registers; +static const int CONTINUE_EXECUTION=-1; + +// +// This function returns either one of EXIT_ codes when it's expected to stop the process or +// CONTINUE_EXECUTION when it's expected to continue further. +// +static int AppInitRawTx(int argc, char* argv[]) { // // Parameters @@ -43,7 +46,7 @@ static bool AppInitRawTx(int argc, char* argv[]) SelectParams(ChainNameFromCommandLine()); } catch (const std::exception& e) { fprintf(stderr, "Error: %s\n", e.what()); - return false; + return EXIT_FAILURE; } fCreateBlank = GetBoolArg("-create", false); @@ -89,57 +92,61 @@ static bool AppInitRawTx(int argc, char* argv[]) strUsage += HelpMessageOpt("set=NAME:JSON-STRING", _("Set register NAME to given JSON-STRING")); fprintf(stdout, "%s", strUsage.c_str()); - return false; + if (argc < 2) { + fprintf(stderr, "Error: too few parameters\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; } - return true; + return CONTINUE_EXECUTION; } -static void RegisterSetJson(const string& key, const string& rawJson) +static void RegisterSetJson(const std::string& key, const std::string& rawJson) { UniValue val; if (!val.read(rawJson)) { - string strErr = "Cannot parse JSON for key " + key; - throw runtime_error(strErr); + std::string strErr = "Cannot parse JSON for key " + key; + throw std::runtime_error(strErr); } registers[key] = val; } -static void RegisterSet(const string& strInput) +static void RegisterSet(const std::string& strInput) { // separate NAME:VALUE in string size_t pos = strInput.find(':'); - if ((pos == string::npos) || + if ((pos == std::string::npos) || (pos == 0) || (pos == (strInput.size() - 1))) - throw runtime_error("Register input requires NAME:VALUE"); + throw std::runtime_error("Register input requires NAME:VALUE"); - string key = strInput.substr(0, pos); - string valStr = strInput.substr(pos + 1, string::npos); + std::string key = strInput.substr(0, pos); + std::string valStr = strInput.substr(pos + 1, std::string::npos); RegisterSetJson(key, valStr); } -static void RegisterLoad(const string& strInput) +static void RegisterLoad(const std::string& strInput) { // separate NAME:FILENAME in string size_t pos = strInput.find(':'); - if ((pos == string::npos) || + if ((pos == std::string::npos) || (pos == 0) || (pos == (strInput.size() - 1))) - throw runtime_error("Register load requires NAME:FILENAME"); + throw std::runtime_error("Register load requires NAME:FILENAME"); - string key = strInput.substr(0, pos); - string filename = strInput.substr(pos + 1, string::npos); + std::string key = strInput.substr(0, pos); + std::string filename = strInput.substr(pos + 1, std::string::npos); FILE *f = fopen(filename.c_str(), "r"); if (!f) { - string strErr = "Cannot open file " + filename; - throw runtime_error(strErr); + std::string strErr = "Cannot open file " + filename; + throw std::runtime_error(strErr); } // load file chunks into one big buffer - string valStr; + std::string valStr; while ((!feof(f)) && (!ferror(f))) { char buf[4096]; int bread = fread(buf, 1, sizeof(buf), f); @@ -153,55 +160,55 @@ static void RegisterLoad(const string& strInput) fclose(f); if (error) { - string strErr = "Error reading file " + filename; - throw runtime_error(strErr); + std::string strErr = "Error reading file " + filename; + throw std::runtime_error(strErr); } // evaluate as JSON buffer register RegisterSetJson(key, valStr); } -static void MutateTxVersion(CMutableTransaction& tx, const string& cmdVal) +static void MutateTxVersion(CMutableTransaction& tx, const std::string& cmdVal) { int64_t newVersion = atoi64(cmdVal); if (newVersion < 1 || newVersion > CTransaction::MAX_STANDARD_VERSION) - throw runtime_error("Invalid TX version requested"); + throw std::runtime_error("Invalid TX version requested"); tx.nVersion = (int) newVersion; } -static void MutateTxLocktime(CMutableTransaction& tx, const string& cmdVal) +static void MutateTxLocktime(CMutableTransaction& tx, const std::string& cmdVal) { int64_t newLocktime = atoi64(cmdVal); if (newLocktime < 0LL || newLocktime > 0xffffffffLL) - throw runtime_error("Invalid TX locktime requested"); + throw std::runtime_error("Invalid TX locktime requested"); tx.nLockTime = (unsigned int) newLocktime; } -static void MutateTxAddInput(CMutableTransaction& tx, const string& strInput) +static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInput) { std::vector<std::string> vStrInputParts; boost::split(vStrInputParts, strInput, boost::is_any_of(":")); // separate TXID:VOUT in string if (vStrInputParts.size()<2) - throw runtime_error("TX input missing separator"); + throw std::runtime_error("TX input missing separator"); // extract and validate TXID - string strTxid = vStrInputParts[0]; + std::string strTxid = vStrInputParts[0]; if ((strTxid.size() != 64) || !IsHex(strTxid)) - throw runtime_error("invalid TX input txid"); + throw std::runtime_error("invalid TX input txid"); uint256 txid(uint256S(strTxid)); static const unsigned int minTxOutSz = 9; static const unsigned int maxVout = MAX_BLOCK_BASE_SIZE / minTxOutSz; // extract and validate vout - string strVout = vStrInputParts[1]; + std::string strVout = vStrInputParts[1]; int vout = atoi(strVout); if ((vout < 0) || (vout > (int)maxVout)) - throw runtime_error("invalid TX input vout"); + throw std::runtime_error("invalid TX input vout"); // extract the optional sequence number uint32_t nSequenceIn=std::numeric_limits<unsigned int>::max(); @@ -213,26 +220,26 @@ static void MutateTxAddInput(CMutableTransaction& tx, const string& strInput) tx.vin.push_back(txin); } -static void MutateTxAddOutAddr(CMutableTransaction& tx, const string& strInput) +static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strInput) { // separate VALUE:ADDRESS in string size_t pos = strInput.find(':'); - if ((pos == string::npos) || + if ((pos == std::string::npos) || (pos == 0) || (pos == (strInput.size() - 1))) - throw runtime_error("TX output missing separator"); + throw std::runtime_error("TX output missing separator"); // extract and validate VALUE - string strValue = strInput.substr(0, pos); + std::string strValue = strInput.substr(0, pos); CAmount value; if (!ParseMoney(strValue, value)) - throw runtime_error("invalid TX output value"); + throw std::runtime_error("invalid TX output value"); // extract and validate ADDRESS - string strAddr = strInput.substr(pos + 1, string::npos); + std::string strAddr = strInput.substr(pos + 1, std::string::npos); CBitcoinAddress addr(strAddr); if (!addr.IsValid()) - throw runtime_error("invalid TX output address"); + throw std::runtime_error("invalid TX output address"); // build standard output script via GetScriptForDestination() CScript scriptPubKey = GetScriptForDestination(addr.Get()); @@ -242,7 +249,7 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const string& strInput) tx.vout.push_back(txout); } -static void MutateTxAddOutData(CMutableTransaction& tx, const string& strInput) +static void MutateTxAddOutData(CMutableTransaction& tx, const std::string& strInput) { CAmount value = 0; @@ -250,20 +257,20 @@ static void MutateTxAddOutData(CMutableTransaction& tx, const string& strInput) size_t pos = strInput.find(':'); if (pos==0) - throw runtime_error("TX output value not specified"); + throw std::runtime_error("TX output value not specified"); - if (pos != string::npos) { + if (pos != std::string::npos) { // extract and validate VALUE - string strValue = strInput.substr(0, pos); + std::string strValue = strInput.substr(0, pos); if (!ParseMoney(strValue, value)) - throw runtime_error("invalid TX output value"); + throw std::runtime_error("invalid TX output value"); } // extract and validate DATA - string strData = strInput.substr(pos + 1, string::npos); + std::string strData = strInput.substr(pos + 1, std::string::npos); if (!IsHex(strData)) - throw runtime_error("invalid TX output data"); + throw std::runtime_error("invalid TX output data"); std::vector<unsigned char> data = ParseHex(strData); @@ -271,22 +278,22 @@ static void MutateTxAddOutData(CMutableTransaction& tx, const string& strInput) tx.vout.push_back(txout); } -static void MutateTxAddOutScript(CMutableTransaction& tx, const string& strInput) +static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& strInput) { // separate VALUE:SCRIPT in string size_t pos = strInput.find(':'); - if ((pos == string::npos) || + if ((pos == std::string::npos) || (pos == 0)) - throw runtime_error("TX output missing separator"); + throw std::runtime_error("TX output missing separator"); // extract and validate VALUE - string strValue = strInput.substr(0, pos); + std::string strValue = strInput.substr(0, pos); CAmount value; if (!ParseMoney(strValue, value)) - throw runtime_error("invalid TX output value"); + throw std::runtime_error("invalid TX output value"); // extract and validate script - string strScript = strInput.substr(pos + 1, string::npos); + std::string strScript = strInput.substr(pos + 1, std::string::npos); CScript scriptPubKey = ParseScript(strScript); // throws on err // construct TxOut, append to transaction output list @@ -294,26 +301,26 @@ static void MutateTxAddOutScript(CMutableTransaction& tx, const string& strInput tx.vout.push_back(txout); } -static void MutateTxDelInput(CMutableTransaction& tx, const string& strInIdx) +static void MutateTxDelInput(CMutableTransaction& tx, const std::string& strInIdx) { // parse requested deletion index int inIdx = atoi(strInIdx); if (inIdx < 0 || inIdx >= (int)tx.vin.size()) { - string strErr = "Invalid TX input index '" + strInIdx + "'"; - throw runtime_error(strErr.c_str()); + std::string strErr = "Invalid TX input index '" + strInIdx + "'"; + throw std::runtime_error(strErr.c_str()); } // delete input from transaction tx.vin.erase(tx.vin.begin() + inIdx); } -static void MutateTxDelOutput(CMutableTransaction& tx, const string& strOutIdx) +static void MutateTxDelOutput(CMutableTransaction& tx, const std::string& strOutIdx) { // parse requested deletion index int outIdx = atoi(strOutIdx); if (outIdx < 0 || outIdx >= (int)tx.vout.size()) { - string strErr = "Invalid TX output index '" + strOutIdx + "'"; - throw runtime_error(strErr.c_str()); + std::string strErr = "Invalid TX output index '" + strOutIdx + "'"; + throw std::runtime_error(strErr.c_str()); } // delete output from transaction @@ -333,7 +340,7 @@ static const struct { {"SINGLE|ANYONECANPAY", SIGHASH_SINGLE|SIGHASH_ANYONECANPAY}, }; -static bool findSighashFlags(int& flags, const string& flagStr) +static bool findSighashFlags(int& flags, const std::string& flagStr) { flags = 0; @@ -347,17 +354,17 @@ static bool findSighashFlags(int& flags, const string& flagStr) return false; } -uint256 ParseHashUO(map<string,UniValue>& o, string strKey) +uint256 ParseHashUO(std::map<std::string,UniValue>& o, std::string strKey) { if (!o.count(strKey)) return uint256(); return ParseHashUV(o[strKey], strKey); } -vector<unsigned char> ParseHexUO(map<string,UniValue>& o, string strKey) +std::vector<unsigned char> ParseHexUO(std::map<std::string,UniValue>& o, std::string strKey) { if (!o.count(strKey)) { - vector<unsigned char> emptyVec; + std::vector<unsigned char> emptyVec; return emptyVec; } return ParseHexUV(o[strKey], strKey); @@ -366,24 +373,24 @@ vector<unsigned char> ParseHexUO(map<string,UniValue>& o, string strKey) static CAmount AmountFromValue(const UniValue& value) { if (!value.isNum() && !value.isStr()) - throw runtime_error("Amount is not a number or string"); + throw std::runtime_error("Amount is not a number or string"); CAmount amount; if (!ParseFixedPoint(value.getValStr(), 8, &amount)) - throw runtime_error("Invalid amount"); + throw std::runtime_error("Invalid amount"); if (!MoneyRange(amount)) - throw runtime_error("Amount out of range"); + throw std::runtime_error("Amount out of range"); return amount; } -static void MutateTxSign(CMutableTransaction& tx, const string& flagStr) +static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) { int nHashType = SIGHASH_ALL; if (flagStr.size() > 0) if (!findSighashFlags(nHashType, flagStr)) - throw runtime_error("unknown sighash flag/sign option"); + throw std::runtime_error("unknown sighash flag/sign option"); - vector<CTransaction> txVariants; + std::vector<CTransaction> txVariants; txVariants.push_back(tx); // mergedTx will end up with all the signatures; it @@ -394,17 +401,17 @@ static void MutateTxSign(CMutableTransaction& tx, const string& flagStr) CCoinsViewCache view(&viewDummy); if (!registers.count("privatekeys")) - throw runtime_error("privatekeys register variable must be set."); + throw std::runtime_error("privatekeys register variable must be set."); CBasicKeyStore tempKeystore; UniValue keysObj = registers["privatekeys"]; for (unsigned int kidx = 0; kidx < keysObj.size(); kidx++) { if (!keysObj[kidx].isStr()) - throw runtime_error("privatekey not a string"); + throw std::runtime_error("privatekey not a std::string"); CBitcoinSecret vchSecret; bool fGood = vchSecret.SetString(keysObj[kidx].getValStr()); if (!fGood) - throw runtime_error("privatekey not valid"); + throw std::runtime_error("privatekey not valid"); CKey key = vchSecret.GetKey(); tempKeystore.AddKey(key); @@ -412,34 +419,34 @@ static void MutateTxSign(CMutableTransaction& tx, const string& flagStr) // Add previous txouts given in the RPC call: if (!registers.count("prevtxs")) - throw runtime_error("prevtxs register variable must be set."); + throw std::runtime_error("prevtxs register variable must be set."); UniValue prevtxsObj = registers["prevtxs"]; { for (unsigned int previdx = 0; previdx < prevtxsObj.size(); previdx++) { UniValue prevOut = prevtxsObj[previdx]; if (!prevOut.isObject()) - throw runtime_error("expected prevtxs internal object"); + throw std::runtime_error("expected prevtxs internal object"); - map<string,UniValue::VType> types = boost::assign::map_list_of("txid", UniValue::VSTR)("vout",UniValue::VNUM)("scriptPubKey",UniValue::VSTR); + std::map<std::string,UniValue::VType> types = boost::assign::map_list_of("txid", UniValue::VSTR)("vout",UniValue::VNUM)("scriptPubKey",UniValue::VSTR); if (!prevOut.checkObject(types)) - throw runtime_error("prevtxs internal object typecheck fail"); + throw std::runtime_error("prevtxs internal object typecheck fail"); uint256 txid = ParseHashUV(prevOut["txid"], "txid"); int nOut = atoi(prevOut["vout"].getValStr()); if (nOut < 0) - throw runtime_error("vout must be positive"); + throw std::runtime_error("vout must be positive"); - vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); + std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { CCoinsModifier coins = view.ModifyCoins(txid); if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { - string err("Previous output scriptPubKey mismatch:\n"); + std::string err("Previous output scriptPubKey mismatch:\n"); err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); - throw runtime_error(err); + throw std::runtime_error(err); } if ((unsigned int)nOut >= coins->vout.size()) coins->vout.resize(nOut+1); @@ -455,7 +462,7 @@ static void MutateTxSign(CMutableTransaction& tx, const string& flagStr) if ((scriptPubKey.IsPayToScriptHash() || scriptPubKey.IsPayToWitnessScriptHash()) && prevOut.exists("redeemScript")) { UniValue v = prevOut["redeemScript"]; - vector<unsigned char> rsData(ParseHexUV(v, "redeemScript")); + std::vector<unsigned char> rsData(ParseHexUV(v, "redeemScript")); CScript redeemScript(rsData.begin(), rsData.end()); tempKeystore.AddCScript(redeemScript); } @@ -512,8 +519,8 @@ public: } }; -static void MutateTx(CMutableTransaction& tx, const string& command, - const string& commandVal) +static void MutateTx(CMutableTransaction& tx, const std::string& command, + const std::string& commandVal) { std::unique_ptr<Secp256k1Init> ecc; @@ -548,7 +555,7 @@ static void MutateTx(CMutableTransaction& tx, const string& command, RegisterSet(commandVal); else - throw runtime_error("unknown command"); + throw std::runtime_error("unknown command"); } static void OutputTxJSON(const CTransaction& tx) @@ -556,20 +563,20 @@ static void OutputTxJSON(const CTransaction& tx) UniValue entry(UniValue::VOBJ); TxToUniv(tx, uint256(), entry); - string jsonOutput = entry.write(4); + std::string jsonOutput = entry.write(4); fprintf(stdout, "%s\n", jsonOutput.c_str()); } static void OutputTxHash(const CTransaction& tx) { - string strHexHash = tx.GetHash().GetHex(); // the hex-encoded transaction hash (aka the transaction id) + std::string strHexHash = tx.GetHash().GetHex(); // the hex-encoded transaction hash (aka the transaction id) fprintf(stdout, "%s\n", strHexHash.c_str()); } static void OutputTxHex(const CTransaction& tx) { - string strHex = EncodeHexTx(tx); + std::string strHex = EncodeHexTx(tx); fprintf(stdout, "%s\n", strHex.c_str()); } @@ -584,10 +591,10 @@ static void OutputTx(const CTransaction& tx) OutputTxHex(tx); } -static string readStdin() +static std::string readStdin() { char buf[4096]; - string ret; + std::string ret; while (!feof(stdin)) { size_t bread = fread(buf, 1, sizeof(buf), stdin); @@ -597,7 +604,7 @@ static string readStdin() } if (ferror(stdin)) - throw runtime_error("error reading stdin"); + throw std::runtime_error("error reading stdin"); boost::algorithm::trim_right(ret); @@ -606,7 +613,7 @@ static string readStdin() static int CommandLineRawTx(int argc, char* argv[]) { - string strPrint; + std::string strPrint; int nRet = 0; try { // Skip switches; Permit common stdin convention "-" @@ -622,15 +629,15 @@ static int CommandLineRawTx(int argc, char* argv[]) if (!fCreateBlank) { // require at least one param if (argc < 2) - throw runtime_error("too few parameters"); + throw std::runtime_error("too few parameters"); // param: hex-encoded bitcoin transaction - string strHexTx(argv[1]); + std::string strHexTx(argv[1]); if (strHexTx == "-") // "-" implies standard input strHexTx = readStdin(); - if (!DecodeHexTx(txDecodeTmp, strHexTx)) - throw runtime_error("invalid transaction encoding"); + if (!DecodeHexTx(txDecodeTmp, strHexTx, true)) + throw std::runtime_error("invalid transaction encoding"); startArg = 2; } else @@ -639,10 +646,10 @@ static int CommandLineRawTx(int argc, char* argv[]) CMutableTransaction tx(txDecodeTmp); for (int i = startArg; i < argc; i++) { - string arg = argv[i]; - string key, value; + std::string arg = argv[i]; + std::string key, value; size_t eqpos = arg.find('='); - if (eqpos == string::npos) + if (eqpos == std::string::npos) key = arg; else { key = arg.substr(0, eqpos); @@ -659,7 +666,7 @@ static int CommandLineRawTx(int argc, char* argv[]) throw; } catch (const std::exception& e) { - strPrint = string("error: ") + e.what(); + strPrint = std::string("error: ") + e.what(); nRet = EXIT_FAILURE; } catch (...) { @@ -678,8 +685,9 @@ int main(int argc, char* argv[]) SetupEnvironment(); try { - if(!AppInitRawTx(argc, argv)) - return EXIT_FAILURE; + int ret = AppInitRawTx(argc, argv); + if (ret != CONTINUE_EXECUTION) + return ret; } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInitRawTx()"); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a89b8cd4ee..ba3ccac615 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -92,7 +92,7 @@ bool AppInit(int argc, char* argv[]) } fprintf(stdout, "%s", strUsage.c_str()); - return false; + return true; } try @@ -126,7 +126,7 @@ bool AppInit(int argc, char* argv[]) if (fCommandLine) { fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); - exit(1); + exit(EXIT_FAILURE); } // -server defaults to true for bitcoind but not for the GUI so do this here SoftSetBoolArg("-server", true); @@ -193,5 +193,5 @@ int main(int argc, char* argv[]) // Connect bitcoind signal handlers noui_connect(); - return (AppInit(argc, argv) ? 0 : 1); + return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 93d3fa372b..2efeda9cc6 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -24,7 +24,7 @@ CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock& block, bool f //TODO: Use our mempool prior to block acceptance to predictively fill more than just the coinbase prefilledtxn[0] = {0, block.vtx[0]}; for (size_t i = 1; i < block.vtx.size(); i++) { - const CTransaction& tx = block.vtx[i]; + const CTransaction& tx = *block.vtx[i]; shorttxids[i - 1] = GetShortID(fUseWTXID ? tx.GetWitnessHash() : tx.GetHash()); } } @@ -59,10 +59,10 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c int32_t lastprefilledindex = -1; for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) { - if (cmpctblock.prefilledtxn[i].tx.IsNull()) + if (cmpctblock.prefilledtxn[i].tx->IsNull()) return READ_STATUS_INVALID; - lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so cant overflow here + lastprefilledindex += cmpctblock.prefilledtxn[i].index + 1; //index is a uint16_t, so can't overflow here if (lastprefilledindex > std::numeric_limits<uint16_t>::max()) return READ_STATUS_INVALID; if ((uint32_t)lastprefilledindex > cmpctblock.shorttxids.size() + i) { @@ -71,7 +71,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c // have neither a prefilled txn or a shorttxid! return READ_STATUS_INVALID; } - txn_available[lastprefilledindex] = std::make_shared<CTransaction>(cmpctblock.prefilledtxn[i].tx); + txn_available[lastprefilledindex] = cmpctblock.prefilledtxn[i].tx; } prefilled_count = cmpctblock.prefilledtxn.size(); @@ -131,7 +131,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c break; } - LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), cmpctblock.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION)); + LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); return READ_STATUS_OK; } @@ -142,7 +142,7 @@ bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { return txn_available[index] ? true : false; } -ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransaction>& vtx_missing) const { +ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing) const { assert(!header.IsNull()); block = header; block.vtx.resize(txn_available.size()); @@ -154,7 +154,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_INVALID; block.vtx[i] = vtx_missing[tx_missing_offset++]; } else - block.vtx[i] = *txn_available[i]; + block.vtx[i] = txn_available[i]; } if (vtx_missing.size() != tx_missing_offset) return READ_STATUS_INVALID; @@ -167,13 +167,13 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< // check its own merkle root and cache that check. if (state.CorruptionPossible()) return READ_STATUS_FAILED; // Possible Short ID collision - return READ_STATUS_INVALID; + return READ_STATUS_CHECKBLOCK_FAILED; } LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", header.GetHash().ToString(), prefilled_count, mempool_count, vtx_missing.size()); if (vtx_missing.size() < 5) { - for(const CTransaction& tx : vtx_missing) - LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx.GetHash().ToString()); + for (const auto& tx : vtx_missing) + LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", header.GetHash().ToString(), tx->GetHash().ToString()); } return READ_STATUS_OK; diff --git a/src/blockencodings.h b/src/blockencodings.h index 99b1cb140d..27baf1f8f8 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -14,14 +14,14 @@ class CTxMemPool; // Dumb helper to handle CTransaction compression at serialize-time struct TransactionCompressor { private: - CTransaction& tx; + CTransactionRef& tx; public: - TransactionCompressor(CTransaction& txIn) : tx(txIn) {} + TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {} ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(tx); //TODO: Compress tx encoding } }; @@ -35,7 +35,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(blockhash); uint64_t indexes_size = (uint64_t)indexes.size(); READWRITE(COMPACTSIZE(indexes_size)); @@ -72,7 +72,7 @@ class BlockTransactions { public: // A BlockTransactions message uint256 blockhash; - std::vector<CTransaction> txn; + std::vector<CTransactionRef> txn; BlockTransactions() {} BlockTransactions(const BlockTransactionsRequest& req) : @@ -81,7 +81,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(blockhash); uint64_t txn_size = (uint64_t)txn.size(); READWRITE(COMPACTSIZE(txn_size)); @@ -104,12 +104,12 @@ struct PrefilledTransaction { // Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs, // as a proper transaction-in-block-index in PartiallyDownloadedBlock uint16_t index; - CTransaction tx; + CTransactionRef tx; ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { uint64_t idx = index; READWRITE(COMPACTSIZE(idx)); if (idx > std::numeric_limits<uint16_t>::max()) @@ -124,6 +124,8 @@ typedef enum ReadStatus_t READ_STATUS_OK, READ_STATUS_INVALID, // Invalid object, peer is sending bogus crap READ_STATUS_FAILED, // Failed to process object + READ_STATUS_CHECKBLOCK_FAILED, // Used only by FillBlock to indicate a + // failure in CheckBlock. } ReadStatus; class CBlockHeaderAndShortTxIDs { @@ -155,7 +157,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(header); READWRITE(nonce); @@ -191,7 +193,7 @@ public: class PartiallyDownloadedBlock { protected: - std::vector<std::shared_ptr<const CTransaction> > txn_available; + std::vector<CTransactionRef> txn_available; size_t prefilled_count = 0, mempool_count = 0; CTxMemPool* pool; public: @@ -200,7 +202,7 @@ public: ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock); bool IsTxAvailable(size_t index) const; - ReadStatus FillBlock(CBlock& block, const std::vector<CTransaction>& vtx_missing) const; + ReadStatus FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing) const; }; #endif diff --git a/src/bloom.cpp b/src/bloom.cpp index 2677652ada..d00befc61c 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -34,7 +34,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int * See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas */ isFull(false), - isEmpty(false), + isEmpty(true), nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), nTweak(nTweakIn), nFlags(nFlagsIn) diff --git a/src/bloom.h b/src/bloom.h index ad6de625d8..d3a017371f 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -73,7 +73,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(vData); READWRITE(nHashFuncs); READWRITE(nTweak); diff --git a/src/chain.h b/src/chain.h index 46a16a3061..0aac5de5c2 100644 --- a/src/chain.h +++ b/src/chain.h @@ -28,7 +28,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(VARINT(nBlocks)); READWRITE(VARINT(nSize)); READWRITE(VARINT(nUndoSize)); @@ -76,7 +76,7 @@ struct CDiskBlockPos ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(VARINT(nFile)); READWRITE(VARINT(nPos)); } @@ -357,8 +357,9 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT(nVersion)); READWRITE(VARINT(nHeight)); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 5850016ae2..3b3c0a5d3e 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -31,7 +31,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi genesis.nBits = nBits; genesis.nNonce = nNonce; genesis.nVersion = nVersion; - genesis.vtx.push_back(txNew); + genesis.vtx.push_back(MakeTransactionRef(std::move(txNew))); genesis.hashPrevBlock.SetNull(); genesis.hashMerkleRoot = BlockMerkleRoot(genesis); return genesis; @@ -96,6 +96,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1479168000; // November 15th, 2016. consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1510704000; // November 15th, 2017. + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000002cb971dd56d1c583c20f90"); + /** * The message start string is designed to be unlikely to occur in normal data. * The characters are rarely used upper ASCII, not valid as UTF-8, and produce @@ -191,6 +194,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1462060800; // May 1st 2016 consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1493596800; // May 1st 2017 + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000000198b4def2baa9338d6"); + pchMessageStart[0] = 0x0b; pchMessageStart[1] = 0x11; pchMessageStart[2] = 0x09; @@ -224,6 +230,7 @@ public: fRequireStandard = false; fMineBlocksOnDemand = false; + checkpointData = (CCheckpointData) { boost::assign::map_list_of ( 546, uint256S("000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")), @@ -265,6 +272,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 0; consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 999999999999ULL; + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x00"); + pchMessageStart[0] = 0xfa; pchMessageStart[1] = 0xbf; pchMessageStart[2] = 0xb5; diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index aefddce464..d22c188c16 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -55,16 +55,6 @@ namespace Checkpoints { return fWorkBefore / (fWorkBefore + fWorkAfter); } - int GetTotalBlocksEstimate(const CCheckpointData& data) - { - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - if (checkpoints.empty()) - return 0; - - return checkpoints.rbegin()->first; - } - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; diff --git a/src/checkpoints.h b/src/checkpoints.h index cd25ea5379..04346f35ff 100644 --- a/src/checkpoints.h +++ b/src/checkpoints.h @@ -19,9 +19,6 @@ struct CCheckpointData; namespace Checkpoints { -//! Return conservative estimate of total number of blocks, 0 if unknown -int GetTotalBlocksEstimate(const CCheckpointData& data); - //! Returns last CBlockIndex* in mapBlockIndex that is a checkpoint CBlockIndex* GetLastCheckpoint(const CCheckpointData& data); diff --git a/src/coins.h b/src/coins.h index 033651a435..d295b3c940 100644 --- a/src/coins.h +++ b/src/coins.h @@ -153,31 +153,8 @@ public: return fCoinBase; } - unsigned int GetSerializeSize(int nType, int nVersion) const { - unsigned int nSize = 0; - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - nSize += ::GetSerializeSize(VARINT(this->nVersion), nType, nVersion); - // size of header code - nSize += ::GetSerializeSize(VARINT(nCode), nType, nVersion); - // spentness bitmask - nSize += nMaskSize; - // txouts themself - for (unsigned int i = 0; i < vout.size(); i++) - if (!vout[i].IsNull()) - nSize += ::GetSerializeSize(CTxOutCompressor(REF(vout[i])), nType, nVersion); - // height - nSize += ::GetSerializeSize(VARINT(nHeight), nType, nVersion); - return nSize; - } - template<typename Stream> - void Serialize(Stream &s, int nType, int nVersion) const { + void Serialize(Stream &s) const { unsigned int nMaskSize = 0, nMaskCode = 0; CalcMaskSize(nMaskSize, nMaskCode); bool fFirst = vout.size() > 0 && !vout[0].IsNull(); @@ -185,33 +162,33 @@ public: assert(fFirst || fSecond || nMaskCode); unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); // version - ::Serialize(s, VARINT(this->nVersion), nType, nVersion); + ::Serialize(s, VARINT(this->nVersion)); // header code - ::Serialize(s, VARINT(nCode), nType, nVersion); + ::Serialize(s, VARINT(nCode)); // spentness bitmask for (unsigned int b = 0; b<nMaskSize; b++) { unsigned char chAvail = 0; for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) if (!vout[2+b*8+i].IsNull()) chAvail |= (1 << i); - ::Serialize(s, chAvail, nType, nVersion); + ::Serialize(s, chAvail); } // txouts themself for (unsigned int i = 0; i < vout.size(); i++) { if (!vout[i].IsNull()) - ::Serialize(s, CTxOutCompressor(REF(vout[i])), nType, nVersion); + ::Serialize(s, CTxOutCompressor(REF(vout[i]))); } // coinbase height - ::Serialize(s, VARINT(nHeight), nType, nVersion); + ::Serialize(s, VARINT(nHeight)); } template<typename Stream> - void Unserialize(Stream &s, int nType, int nVersion) { + void Unserialize(Stream &s) { unsigned int nCode = 0; // version - ::Unserialize(s, VARINT(this->nVersion), nType, nVersion); + ::Unserialize(s, VARINT(this->nVersion)); // header code - ::Unserialize(s, VARINT(nCode), nType, nVersion); + ::Unserialize(s, VARINT(nCode)); fCoinBase = nCode & 1; std::vector<bool> vAvail(2, false); vAvail[0] = (nCode & 2) != 0; @@ -220,7 +197,7 @@ public: // spentness bitmask while (nMaskCode > 0) { unsigned char chAvail = 0; - ::Unserialize(s, chAvail, nType, nVersion); + ::Unserialize(s, chAvail); for (unsigned int p = 0; p < 8; p++) { bool f = (chAvail & (1 << p)) != 0; vAvail.push_back(f); @@ -232,10 +209,10 @@ public: vout.assign(vAvail.size(), CTxOut()); for (unsigned int i = 0; i < vAvail.size(); i++) { if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i])), nType, nVersion); + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); } // coinbase height - ::Unserialize(s, VARINT(nHeight), nType, nVersion); + ::Unserialize(s, VARINT(nHeight)); Cleanup(); } diff --git a/src/compat.h b/src/compat.h index 79a297e5e4..2578d6d344 100644 --- a/src/compat.h +++ b/src/compat.h @@ -34,6 +34,7 @@ #else #include <sys/fcntl.h> #include <sys/mman.h> +#include <sys/select.h> #include <sys/socket.h> #include <sys/types.h> #include <net/if.h> diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 899220bdc5..07ca535728 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin developers +// Copyright (c) 2014 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/endian.h b/src/compat/endian.h index 6bfae42c77..f7c1f9318a 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Bitcoin developers +// Copyright (c) 2014-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.h b/src/compressor.h index fa702f0dfa..961365d261 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -55,16 +55,8 @@ protected: public: CScriptCompressor(CScript &scriptIn) : script(scriptIn) { } - unsigned int GetSerializeSize(int nType, int nVersion) const { - std::vector<unsigned char> compr; - if (Compress(compr)) - return compr.size(); - unsigned int nSize = script.size() + nSpecialScripts; - return script.size() + VARINT(nSize).GetSerializeSize(nType, nVersion); - } - template<typename Stream> - void Serialize(Stream &s, int nType, int nVersion) const { + void Serialize(Stream &s) const { std::vector<unsigned char> compr; if (Compress(compr)) { s << CFlatData(compr); @@ -76,7 +68,7 @@ public: } template<typename Stream> - void Unserialize(Stream &s, int nType, int nVersion) { + void Unserialize(Stream &s) { unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { @@ -112,7 +104,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { if (!ser_action.ForRead()) { uint64_t nVal = CompressAmount(txout.nValue); READWRITE(VARINT(nVal)); diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 35f7d2e05a..6fa96ddf45 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -160,7 +160,7 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated) std::vector<uint256> leaves; leaves.resize(block.vtx.size()); for (size_t s = 0; s < block.vtx.size(); s++) { - leaves[s] = block.vtx[s].GetHash(); + leaves[s] = block.vtx[s]->GetHash(); } return ComputeMerkleRoot(leaves, mutated); } @@ -171,7 +171,7 @@ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated) leaves.resize(block.vtx.size()); leaves[0].SetNull(); // The witness hash of the coinbase is 0. for (size_t s = 1; s < block.vtx.size(); s++) { - leaves[s] = block.vtx[s].GetWitnessHash(); + leaves[s] = block.vtx[s]->GetWitnessHash(); } return ComputeMerkleRoot(leaves, mutated); } @@ -181,7 +181,7 @@ std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position) std::vector<uint256> leaves; leaves.resize(block.vtx.size()); for (size_t s = 0; s < block.vtx.size(); s++) { - leaves[s] = block.vtx[s].GetHash(); + leaves[s] = block.vtx[s]->GetHash(); } return ComputeMerkleBranch(leaves, position); } diff --git a/src/consensus/params.h b/src/consensus/params.h index 0e73cace83..20efc68ade 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -61,6 +61,7 @@ struct Params { int64_t nPowTargetSpacing; int64_t nPowTargetTimespan; int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; } + uint256 nMinimumChainWork; }; } // namespace Consensus diff --git a/src/core_memusage.h b/src/core_memusage.h index b8e0f08bbf..0dcc24c40c 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -69,8 +69,8 @@ static inline size_t RecursiveDynamicUsage(const CMutableTransaction& tx) { static inline size_t RecursiveDynamicUsage(const CBlock& block) { size_t mem = memusage::DynamicUsage(block.vtx); - for (std::vector<CTransaction>::const_iterator it = block.vtx.begin(); it != block.vtx.end(); it++) { - mem += RecursiveDynamicUsage(*it); + for (const auto& tx : block.vtx) { + mem += memusage::DynamicUsage(tx) + RecursiveDynamicUsage(*tx); } return mem; } diff --git a/src/crypto/ctaes/test.c b/src/crypto/ctaes/test.c index fce1696acd..21439a16f1 100644 --- a/src/crypto/ctaes/test.c +++ b/src/crypto/ctaes/test.c @@ -102,7 +102,7 @@ int main(void) { } } if (fail == 0) { - fprintf(stderr, "All tests succesful\n"); + fprintf(stderr, "All tests successful\n"); } else { fprintf(stderr, "%i tests failed\n", fail); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 47bdb31b5b..4a79bbd17d 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -17,6 +17,9 @@ #include <leveldb/db.h> #include <leveldb/write_batch.h> +static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; +static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; + class dbwrapper_error : public std::runtime_error { public: @@ -60,12 +63,12 @@ public: void Write(const K& key, const V& value) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(ssValue.GetSerializeSize(value)); + ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(&ssValue[0], ssValue.size()); @@ -77,7 +80,7 @@ public: void Erase(const K& key) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); @@ -107,7 +110,7 @@ public: template<typename K> void Seek(const K& key) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); piter->Seek(slKey); @@ -200,7 +203,7 @@ public: bool Read(const K& key, V& value) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); @@ -234,7 +237,7 @@ public: bool Exists(const K& key) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(&ssKey[0], ssKey.size()); diff --git a/src/hash.h b/src/hash.h index db4e130ae7..94e7f5ea6c 100644 --- a/src/hash.h +++ b/src/hash.h @@ -132,15 +132,17 @@ class CHashWriter private: CHash256 ctx; + const int nType; + const int nVersion; public: - int nType; - int nVersion; CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} - CHashWriter& write(const char *pch, size_t size) { + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + + void write(const char *pch, size_t size) { ctx.Write((const unsigned char*)pch, size); - return (*this); } // invalidates the object @@ -153,7 +155,7 @@ public: template<typename T> CHashWriter& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } }; diff --git a/src/init.cpp b/src/init.cpp index fb51eb7633..63ec744812 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -207,6 +207,7 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); + DumpMempool(); if (fFeeEstimatesInitialized) { @@ -359,13 +360,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s)")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -noconnect or -connect=0 alone to disable automatic connections")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); - strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect)")); + strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect/-noconnect)")); strUsage += HelpMessageOpt("-externalip=<ip>", _("Specify your own public address")); strUsage += HelpMessageOpt("-forcednsseed", strprintf(_("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); - strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect)")); + strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect/-noconnect)")); strUsage += HelpMessageOpt("-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt("-maxconnections=<n>", strprintf(_("Maintain at most <n> connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=<n>", strprintf(_("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); @@ -600,6 +601,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) { const CChainParams& chainparams = Params(); RenameThread("bitcoin-loadblk"); + + { CImportingNow imp; // -reindex @@ -659,6 +662,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) LogPrintf("Stopping after block import\n"); StartShutdown(); } + } // End scope of CImportingNow + LoadMempool(); } /** Sanity checks @@ -746,23 +751,10 @@ void InitParameterInteraction() LogPrintf("%s: parameter interaction: -externalip set -> setting -discover=0\n", __func__); } - if (GetBoolArg("-salvagewallet", false)) { - // Rewrite just private keys: rescan to find transactions - if (SoftSetBoolArg("-rescan", true)) - LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); - } - - // -zapwallettx implies a rescan - if (GetBoolArg("-zapwallettxes", false)) { - if (SoftSetBoolArg("-rescan", true)) - LogPrintf("%s: parameter interaction: -zapwallettxes=<mode> -> setting -rescan=1\n", __func__); - } - - // disable walletbroadcast and whitelistrelay in blocksonly mode + // disable whitelistrelay in blocksonly mode if (GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { if (SoftSetBoolArg("-whitelistrelay", false)) LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -whitelistrelay=0\n", __func__); - // walletbroadcast is disabled in CWallet::ParameterInteraction() } // Forcing relay from whitelisted hosts implies we will accept relays from them in the first place. @@ -1134,6 +1126,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) return false; #endif // ********************************************************* Step 6: network initialization + // Note that we absolutely cannot open any actual connections + // until the very end ("start node") as the UTXO/block state + // is not yet setup and may end up being set up twice if we + // need to reindex later. assert(!g_connman); g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); @@ -1354,7 +1350,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CleanupBlockRevFiles(); } - if (!LoadBlockIndex()) { + if (!LoadBlockIndex(chainparams)) { strLoadError = _("Error loading block database"); break; } @@ -1527,13 +1523,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.NotifyBlockTip.disconnect(BlockNotifyGenesisWait); } -#ifdef ENABLE_WALLET - // Add wallet transactions that aren't already in a block to mempool - // Do this here as mempool requires genesis block to be loaded - if (pwalletMain) - pwalletMain->ReacceptWalletTransactions(); -#endif - // ********************************************************* Step 11: start node //// debug print @@ -1571,10 +1560,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.InitMessage(_("Done loading")); #ifdef ENABLE_WALLET - if (pwalletMain) { - // Run a thread to flush wallet periodically - threadGroup.create_thread(boost::bind(&ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile))); - } + if (pwalletMain) + pwalletMain->postInitProcess(threadGroup); #endif return !fRequestShutdown; diff --git a/src/key.cpp b/src/key.cpp index aae9b042ac..b3ea98fb92 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -125,8 +125,8 @@ bool CKey::Check(const unsigned char *vch) { void CKey::MakeNewKey(bool fCompressedIn) { do { - GetStrongRandBytes(vch, sizeof(vch)); - } while (!Check(vch)); + GetStrongRandBytes(keydata.data(), keydata.size()); + } while (!Check(keydata.data())); fValid = true; fCompressed = fCompressedIn; } @@ -224,20 +224,18 @@ bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert(IsCompressed()); - unsigned char out[64]; - LockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); assert(pubkey.begin() + 33 == pubkey.end()); - BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, out); + BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(begin() + 32 == end()); - BIP32Hash(cc, nChild, 0, begin(), out); + BIP32Hash(cc, nChild, 0, begin(), vout.data()); } - memcpy(ccChild.begin(), out+32, 32); + memcpy(ccChild.begin(), vout.data()+32, 32); memcpy((unsigned char*)keyChild.begin(), begin(), 32); - bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), out); - UnlockObject(out); + bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); keyChild.fCompressed = true; keyChild.fValid = ret; return ret; @@ -253,12 +251,10 @@ bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; - unsigned char out[64]; - LockObject(out); - CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(out); - key.Set(&out[0], &out[32], true); - memcpy(chaincode.begin(), &out[32], 32); - UnlockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); + CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); + key.Set(&vout[0], &vout[32], true); + memcpy(chaincode.begin(), &vout[32], 32); nDepth = 0; nChild = 0; memset(vchFingerprint, 0, sizeof(vchFingerprint)); @@ -308,12 +304,10 @@ void ECC_Start() { { // Pass in a random blinding seed to the secp256k1 context. - unsigned char seed[32]; - LockObject(seed); - GetRandBytes(seed, 32); - bool ret = secp256k1_context_randomize(ctx, seed); + std::vector<unsigned char, secure_allocator<unsigned char>> vseed(32); + GetRandBytes(vseed.data(), 32); + bool ret = secp256k1_context_randomize(ctx, vseed.data()); assert(ret); - UnlockObject(seed); } secp256k1_context_sign = ctx; @@ -43,9 +43,7 @@ private: bool fCompressed; //! The actual byte data - unsigned char vch[32]; - - static_assert(sizeof(vch) == 32, "vch must be 32 bytes in length to not break serialization"); + std::vector<unsigned char, secure_allocator<unsigned char> > keydata; //! Check whether the 32-byte array pointed to be vch is valid keydata. bool static Check(const unsigned char* vch); @@ -54,37 +52,30 @@ public: //! Construct an invalid private key. CKey() : fValid(false), fCompressed(false) { - LockObject(vch); - } - - //! Copy constructor. This is necessary because of memlocking. - CKey(const CKey& secret) : fValid(secret.fValid), fCompressed(secret.fCompressed) - { - LockObject(vch); - memcpy(vch, secret.vch, sizeof(vch)); + // Important: vch must be 32 bytes in length to not break serialization + keydata.resize(32); } //! Destructor (again necessary because of memlocking). ~CKey() { - UnlockObject(vch); } friend bool operator==(const CKey& a, const CKey& b) { return a.fCompressed == b.fCompressed && a.size() == b.size() && - memcmp(&a.vch[0], &b.vch[0], a.size()) == 0; + memcmp(a.keydata.data(), b.keydata.data(), a.size()) == 0; } //! Initialize using begin and end iterators to byte data. template <typename T> void Set(const T pbegin, const T pend, bool fCompressedIn) { - if (pend - pbegin != sizeof(vch)) { + if (size_t(pend - pbegin) != keydata.size()) { fValid = false; } else if (Check(&pbegin[0])) { - memcpy(vch, (unsigned char*)&pbegin[0], sizeof(vch)); + memcpy(keydata.data(), (unsigned char*)&pbegin[0], keydata.size()); fValid = true; fCompressed = fCompressedIn; } else { @@ -93,9 +84,9 @@ public: } //! Simple read-only vector-like interface. - unsigned int size() const { return (fValid ? sizeof(vch) : 0); } - const unsigned char* begin() const { return vch; } - const unsigned char* end() const { return vch + size(); } + unsigned int size() const { return (fValid ? keydata.size() : 0); } + const unsigned char* begin() const { return keydata.data(); } + const unsigned char* end() const { return keydata.data() + size(); } //! Check whether this private key is valid. bool IsValid() const { return fValid; } @@ -171,7 +162,7 @@ struct CExtKey { CExtPubKey Neuter() const; void SetMaster(const unsigned char* seed, unsigned int nSeedLen); template <typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { unsigned int len = BIP32_EXTKEY_SIZE; ::WriteCompactSize(s, len); @@ -180,7 +171,7 @@ struct CExtKey { s.write((const char *)&code[0], len); } template <typename Stream> - void Unserialize(Stream& s, int nType, int nVersion) + void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); unsigned char code[BIP32_EXTKEY_SIZE]; diff --git a/src/main.cpp b/src/main.cpp index 9c0f25950a..452ceac568 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,6 +18,8 @@ #include "init.h" #include "merkleblock.h" #include "net.h" +#include "netmessagemaker.h" +#include "netbase.h" #include "policy/fees.h" #include "policy/policy.h" #include "pow.h" @@ -63,11 +65,11 @@ CCriticalSection cs_main; BlockMap mapBlockIndex; CChain chainActive; CBlockIndex *pindexBestHeader = NULL; -int64_t nTimeBestReceived = 0; +int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we last received a block CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; int nScriptCheckThreads = 0; -bool fImporting = false; +std::atomic_bool fImporting(false); bool fReindex = false; bool fTxIndex = false; bool fHavePruned = false; @@ -179,8 +181,10 @@ namespace { * Sources of received blocks, saved to be able to send them reject * messages or ban them when processing happens afterwards. Protected by * cs_main. + * Set mapBlockSource[hash].second to false if the node should not be + * punished if the block is invalid. */ - map<uint256, NodeId> mapBlockSource; + map<uint256, std::pair<NodeId, bool>> mapBlockSource; /** * Filter for transactions that were recently rejected by @@ -230,7 +234,7 @@ namespace { int nPeersWithValidatedDownloads = 0; /** Relay map, protected by cs_main. */ - typedef std::map<uint256, std::shared_ptr<const CTransaction>> MapRelay; + typedef std::map<uint256, CTransactionRef> MapRelay; MapRelay mapRelay; /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration; @@ -257,7 +261,7 @@ struct CBlockReject { */ struct CNodeState { //! The peer's address - CService address; + const CService address; //! Whether we have a fully established connection. bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. @@ -265,7 +269,7 @@ struct CNodeState { //! Whether this peer should be disconnected and banned (unless whitelisted). bool fShouldBan; //! String name of this peer (debugging/logging purposes). - std::string name; + const std::string name; //! List of asynchronously-determined block rejections to notify this peer about. std::vector<CBlockReject> rejects; //! The best known block we know this peer has announced. @@ -309,7 +313,7 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; - CNodeState() { + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; fShouldBan = false; @@ -354,11 +358,36 @@ void UpdatePreferredDownload(CNode* node, CNodeState* state) nPreferredDownload += state->fPreferredDownload; } -void InitializeNode(NodeId nodeid, const CNode *pnode) { - LOCK(cs_main); - CNodeState &state = mapNodeState.insert(std::make_pair(nodeid, CNodeState())).first->second; - state.name = pnode->addrName; - state.address = pnode->addr; +void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) +{ + ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); + uint64_t nonce = pnode->GetLocalNonce(); + int nNodeStartingHeight = pnode->GetMyStartingHeight(); + NodeId nodeid = pnode->GetId(); + CAddress addr = pnode->addr; + + CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); + CAddress addrMe = CAddress(CService(), nLocalNodeServices); + + connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); + + if (fLogIPs) + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + else + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); +} + +void InitializeNode(CNode *pnode, CConnman& connman) { + CAddress addr = pnode->addr; + std::string addrName = pnode->addrName; + NodeId nodeid = pnode->GetId(); + { + LOCK(cs_main); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + } + if(!pnode->fInbound) + PushNodeVersion(pnode, connman, GetTime()); } void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { @@ -501,15 +530,14 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pf if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - bool found = connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - pnodeStop->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); return true; }); - if(found) - lNodesAnnouncingHeaderAndIDs.pop_front(); + lNodesAnnouncingHeaderAndIDs.pop_front(); } fAnnounceUsingCMPCTBLOCK = true; - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); } } @@ -691,6 +719,16 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc CCoinsViewCache *pcoinsTip = NULL; CBlockTreeDB *pblocktree = NULL; +enum FlushStateMode { + FLUSH_STATE_NONE, + FLUSH_STATE_IF_NEEDED, + FLUSH_STATE_PERIODIC, + FLUSH_STATE_ALWAYS +}; + +// See definition for documentation +bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode); + ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -1066,7 +1104,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i -bool CheckTransaction(const CTransaction& tx, CValidationState &state) +bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) { // Basic checks that don't depend on any context if (tx.vin.empty()) @@ -1090,13 +1128,14 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); } - // Check for duplicate inputs - set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (vInOutPoints.count(txin.prevout)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - vInOutPoints.insert(txin.prevout); + // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock + if (fCheckDuplicateInputs) { + set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) + { + if (!vInOutPoints.insert(txin.prevout).second) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + } } if (tx.IsCoinBase()) @@ -1135,7 +1174,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) { const uint256 hash = tx.GetHash(); @@ -1308,7 +1347,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); + CTxMemPoolEntry entry(tx, nFees, nAcceptTime, dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of @@ -1572,18 +1611,27 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return true; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); } + // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits + CValidationState stateDummy; + FlushStateToDisk(stateDummy, FLUSH_STATE_PERIODIC); return res; } +bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +{ + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee); +} + /** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -1591,7 +1639,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P LOCK(cs_main); - std::shared_ptr<const CTransaction> ptx = mempool.get(hash); + CTransactionRef ptx = mempool.get(hash); if (ptx) { txOut = *ptx; @@ -1634,9 +1682,9 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P if (pindexSlow) { CBlock block; if (ReadBlockFromDisk(block, pindexSlow, consensusParams)) { - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - if (tx.GetHash() == hash) { - txOut = tx; + for (const auto& tx : block.vtx) { + if (tx->GetHash() == hash) { + txOut = *tx; hashBlock = pindexSlow->GetBlockHash(); return true; } @@ -1665,7 +1713,7 @@ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHea return error("WriteBlockToDisk: OpenBlockFile failed"); // Write index header - unsigned int nSize = fileout.GetSerializeSize(block); + unsigned int nSize = GetSerializeSize(fileout, block); fileout << FLATDATA(messageStart) << nSize; // Write block @@ -1740,13 +1788,14 @@ bool IsInitialBlockDownload() return false; if (fImporting || fReindex) return true; - if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) + if (chainActive.Tip() == NULL) + return true; + if (chainActive.Tip()->nChainWork < UintToArith256(chainParams.GetConsensus().nMinimumChainWork)) + return true; + if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; - bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || - std::max(chainActive.Tip()->GetBlockTime(), pindexBestHeader->GetBlockTime()) < GetTime() - nMaxTipAge); - if (!state) - latchToFalse.store(true, std::memory_order_relaxed); - return state; + latchToFalse.store(true, std::memory_order_relaxed); + return false; } bool fLargeWorkForkFound = false; @@ -1774,7 +1823,7 @@ void CheckForkWarningConditions() { AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks - // (we assume we don't get stuck on a fork before the last checkpoint) + // (we assume we don't get stuck on a fork before finishing our initial sync) if (IsInitialBlockDownload()) return; @@ -2054,7 +2103,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return error("%s: OpenUndoFile failed", __func__); // Write index header - unsigned int nSize = fileout.GetSerializeSize(blockundo); + unsigned int nSize = GetSerializeSize(fileout, blockundo); fileout << FLATDATA(messageStart) << nSize; // Write undo data @@ -2174,7 +2223,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { - const CTransaction &tx = block.vtx[i]; + const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); // Check that all outputs are available and match the outputs in the block itself @@ -2368,8 +2417,8 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == chainparams.GetConsensus().BIP34Hash)); if (fEnforceBIP30) { - BOOST_FOREACH(const CTransaction& tx, block.vtx) { - const CCoins* coins = view.AccessCoins(tx.GetHash()); + for (const auto& tx : block.vtx) { + const CCoins* coins = view.AccessCoins(tx->GetHash()); if (coins && !coins->IsPruned()) return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); @@ -2412,7 +2461,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); - std::vector<uint256> vOrphanErase; std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; @@ -2425,7 +2473,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated for (unsigned int i = 0; i < block.vtx.size(); i++) { - const CTransaction &tx = block.vtx[i]; + const CTransaction &tx = *(block.vtx[i]); nInputs += tx.vin.size(); @@ -2443,17 +2491,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; } - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = (*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); - } - } - if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); @@ -2495,10 +2532,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); - if (block.vtx[0].GetValueOut() > blockReward) + if (block.vtx[0]->GetValueOut() > blockReward) return state.DoS(100, error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0].GetValueOut(), blockReward), + block.vtx[0]->GetValueOut(), blockReward), REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) @@ -2541,16 +2578,8 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // Watch for changes to the previous coinbase transaction. static uint256 hashPrevBestCoinBase; GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0].GetHash(); + hashPrevBestCoinBase = block.vtx[0]->GetHash(); - // Erase orphan transactions include or precluded by this block - if (vOrphanErase.size()) { - int nErased = 0; - BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { - nErased += EraseOrphanTx(orphanHash); - } - LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); - } int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -2558,13 +2587,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return true; } -enum FlushStateMode { - FLUSH_STATE_NONE, - FLUSH_STATE_IF_NEEDED, - FLUSH_STATE_PERIODIC, - FLUSH_STATE_ALWAYS -}; - /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -2684,7 +2706,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); // New best block - nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); cvBlockChange.notify_all(); @@ -2766,12 +2787,12 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara if (!fBare) { // Resurrect mempool transactions from the disconnected block. std::vector<uint256> vHashUpdate; - BOOST_FOREACH(const CTransaction &tx, block.vtx) { + for (const auto& it : block.vtx) { + const CTransaction& tx = *it; // ignore validation errors in resurrected transactions - list<CTransaction> removed; CValidationState stateDummy; if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { - mempool.removeRecursive(tx, removed); + mempool.removeRecursive(tx); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); } @@ -2788,8 +2809,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - GetMainSignals().SyncTransaction(tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + for (const auto& tx : block.vtx) { + GetMainSignals().SyncTransaction(*tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); } return true; } @@ -2804,7 +2825,7 @@ static int64_t nTimePostConnect = 0; * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::list<CTransaction> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>> &txChanged) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::vector<CTransactionRef> &txConflicted, std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>> &txChanged) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2840,11 +2861,11 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, &txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); - for(unsigned int i=0; i < pblock->vtx.size(); i++) + for (unsigned int i=0; i < pblock->vtx.size(); i++) txChanged.emplace_back(pblock->vtx[i], pindexNew, i); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; @@ -2927,7 +2948,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::list<CTransaction>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>>& txChanged) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::vector<CTransactionRef>& txConflicted, std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>>& txChanged) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -3028,7 +3049,7 @@ static void NotifyHeaderTip() { bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, const CBlock *pblock) { CBlockIndex *pindexMostWork = NULL; CBlockIndex *pindexNewTip = NULL; - std::vector<std::tuple<CTransaction,CBlockIndex*,int>> txChanged; + std::vector<std::tuple<CTransactionRef,CBlockIndex*,int>> txChanged; if (pblock) txChanged.reserve(pblock->vtx.size()); do { @@ -3038,7 +3059,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - std::list<CTransaction> txConflicted; + std::vector<CTransactionRef> txConflicted; bool fInitialDownload; { LOCK(cs_main); @@ -3069,13 +3090,13 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, // throw all transactions though the signal-interface // while _not_ holding the cs_main lock - BOOST_FOREACH(const CTransaction &tx, txConflicted) + for (const auto& tx : txConflicted) { - GetMainSignals().SyncTransaction(tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().SyncTransaction(*tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); } // ... and about transactions that got confirmed: - for(unsigned int i = 0; i < txChanged.size(); i++) - GetMainSignals().SyncTransaction(std::get<0>(txChanged[i]), std::get<1>(txChanged[i]), std::get<2>(txChanged[i])); + for (unsigned int i = 0; i < txChanged.size(); i++) + GetMainSignals().SyncTransaction(*std::get<0>(txChanged[i]), std::get<1>(txChanged[i]), std::get<2>(txChanged[i])); // Notify external listeners about the new tip. GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); @@ -3161,6 +3182,7 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C InvalidChainFound(pindex); mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } @@ -3414,22 +3436,22 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); // First transaction must be coinbase, the rest must not be - if (block.vtx.empty() || !block.vtx[0].IsCoinBase()) + if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) - if (block.vtx[i].IsCoinBase()) + if (block.vtx[i]->IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); // Check transactions for (const auto& tx : block.vtx) - if (!CheckTransaction(tx, state)) + if (!CheckTransaction(*tx, state, false)) return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), - strprintf("Transaction check failed (tx hash %s) %s", tx.GetHash().ToString(), state.GetDebugMessage())); + strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), state.GetDebugMessage())); unsigned int nSigOps = 0; for (const auto& tx : block.vtx) { - nSigOps += GetLegacySigOpCount(tx); + nSigOps += GetLegacySigOpCount(*tx); } if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); @@ -3465,8 +3487,8 @@ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& pa static int GetWitnessCommitmentIndex(const CBlock& block) { int commitpos = -1; - for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { - if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { + if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { commitpos = o; } } @@ -3477,10 +3499,12 @@ void UpdateUncommittedBlockStructures(CBlock& block, const CBlockIndex* pindexPr { int commitpos = GetWitnessCommitmentIndex(block); static const std::vector<unsigned char> nonce(32, 0x00); - if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0].wit.IsEmpty()) { - block.vtx[0].wit.vtxinwit.resize(1); - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.resize(1); - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = nonce; + if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0]->wit.IsEmpty()) { + CMutableTransaction tx(*block.vtx[0]); + tx.wit.vtxinwit.resize(1); + tx.wit.vtxinwit[0].scriptWitness.stack.resize(1); + tx.wit.vtxinwit[0].scriptWitness.stack[0] = nonce; + block.vtx[0] = MakeTransactionRef(std::move(tx)); } } @@ -3488,15 +3512,8 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc { std::vector<unsigned char> commitment; int commitpos = GetWitnessCommitmentIndex(block); - bool fHaveWitness = false; - for (size_t t = 1; t < block.vtx.size(); t++) { - if (!block.vtx[t].wit.IsNull()) { - fHaveWitness = true; - break; - } - } std::vector<unsigned char> ret(32, 0x00); - if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) { + if (consensusParams.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { if (commitpos == -1) { uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); @@ -3511,8 +3528,8 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc out.scriptPubKey[5] = 0xed; memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); - const_cast<std::vector<CTxOut>*>(&block.vtx[0].vout)->push_back(out); - block.vtx[0].UpdateHash(); + const_cast<std::vector<CTxOut>*>(&block.vtx[0]->vout)->push_back(out); + block.vtx[0]->UpdateHash(); } } UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); @@ -3561,7 +3578,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // Check that all transactions are finalized for (const auto& tx : block.vtx) { - if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { + if (!IsFinalTx(*tx, nHeight, nLockTimeCutoff)) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); } } @@ -3570,8 +3587,8 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co if (nHeight >= consensusParams.BIP34Height) { CScript expect = CScript() << nHeight; - if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || - !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) { + if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || + !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { return state.DoS(100, false, REJECT_INVALID, "bad-cb-height", false, "block height mismatch in coinbase"); } } @@ -3593,11 +3610,11 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // The malleation check is ignored; as the transaction tree itself // already does not permit it, it is impossible to trigger in the // witness tree. - if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { + if (block.vtx[0]->wit.vtxinwit.size() != 1 || block.vtx[0]->wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0]->wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { return state.DoS(100, false, REJECT_INVALID, "bad-witness-nonce-size", true, strprintf("%s : invalid witness nonce size", __func__)); } - CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); - if (memcmp(hashWitness.begin(), &block.vtx[0].vout[commitpos].scriptPubKey[6], 32)) { + CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0]->wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); + if (memcmp(hashWitness.begin(), &block.vtx[0]->vout[commitpos].scriptPubKey[6], 32)) { return state.DoS(100, false, REJECT_INVALID, "bad-witness-merkle-match", true, strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; @@ -3607,7 +3624,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { for (size_t i = 0; i < block.vtx.size(); i++) { - if (!block.vtx[i].wit.IsNull()) { + if (!block.vtx[i]->wit.IsNull()) { return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } @@ -3670,6 +3687,8 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (ppindex) *ppindex = pindex; + CheckBlockIndex(chainparams.GetConsensus()); + return true; } @@ -3697,6 +3716,11 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha // not process unrequested blocks. bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); + // TODO: Decouple this function from the block download logic by removing fRequested + // This requires some new chain datastructure to efficiently look up if a + // block is in a chain leading to a candidate for best tip, despite not + // being such a candidate itself. + // TODO: deal better with return value and error conditions for duplicate // and unrequested blocks. if (fAlreadyHave) return true; @@ -3741,28 +3765,26 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha return true; } -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) +bool ProcessNewBlock(const CChainParams& chainparams, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp, bool *fNewBlock) { { LOCK(cs_main); - bool fRequested = MarkBlockAsReceived(pblock->GetHash()); - fRequested |= fForceProcessing; // Store to disk CBlockIndex *pindex = NULL; - bool fNewBlock = false; - bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp, &fNewBlock); - if (pindex && pfrom) { - mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); - if (fNewBlock) pfrom->nLastBlockTime = GetTime(); - } + if (fNewBlock) *fNewBlock = false; + CValidationState state; + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fForceProcessing, dbp, fNewBlock); CheckBlockIndex(chainparams.GetConsensus()); - if (!ret) + if (!ret) { + GetMainSignals().BlockChecked(*pblock, state); return error("%s: AcceptBlock FAILED", __func__); + } } NotifyHeaderTip(); + CValidationState state; // Only used to report errors, not invalidity - ignore it if (!ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed", __func__); @@ -3967,9 +3989,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash) return pindexNew; } -bool static LoadBlockIndexDB() +bool static LoadBlockIndexDB(const CChainParams& chainparams) { - const CChainParams& chainparams = Params(); if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; @@ -4264,6 +4285,9 @@ bool RewindBlockIndex(const CChainParams& params) return true; } +// May NOT be used after any connections are up as much +// of the peer-processing logic assumes a consistent +// block index state void UnloadBlockIndex() { LOCK(cs_main); @@ -4274,18 +4298,12 @@ void UnloadBlockIndex() mempool.clear(); mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); - nSyncStarted = 0; mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; nBlockSequenceId = 1; - mapBlockSource.clear(); - mapBlocksInFlight.clear(); - nPreferredDownload = 0; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); - mapNodeState.clear(); - recentRejects.reset(NULL); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); @@ -4298,10 +4316,10 @@ void UnloadBlockIndex() fHavePruned = false; } -bool LoadBlockIndex() +bool LoadBlockIndex(const CChainParams& chainparams) { // Load block index from databases - if (!fReindex && !LoadBlockIndexDB()) + if (!fReindex && !LoadBlockIndexDB(chainparams)) return false; return true; } @@ -4310,9 +4328,6 @@ bool InitBlockIndex(const CChainParams& chainparams) { LOCK(cs_main); - // Initialize global variables that cannot be constructed at startup. - recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); - // Check whether we're already initialized if (chainActive.Genesis() != NULL) return true; @@ -4701,6 +4716,39 @@ std::string GetWarnings(const std::string& strFor) // blockchain -> download logic notification // +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { + // Initialize global variables that cannot be constructed at startup. + recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); +} + +void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { + if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) + return; + + LOCK(cs_main); + + std::vector<uint256> vOrphanErase; + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } + } + + // Erase orphan transactions include or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + } +} + void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; connman->SetBestHeight(nNewHeight); @@ -4727,22 +4775,24 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } }); } + + nTimeBestReceived = GetTime(); } void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { LOCK(cs_main); const uint256 hash(block.GetHash()); - std::map<uint256, NodeId>::iterator it = mapBlockSource.find(hash); + std::map<uint256, std::pair<NodeId, bool>>::iterator it = mapBlockSource.find(hash); int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second)) { + if (it != mapBlockSource.end() && State(it->second.first)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; - State(it->second)->rejects.push_back(reject); - if (nDoS > 0) - Misbehaving(it->second, nDoS); + State(it->second.first)->rejects.push_back(reject); + if (nDoS > 0 && it->second.second) + Misbehaving(it->second.first, nDoS); } } if (it != mapBlockSource.end()) @@ -4799,26 +4849,35 @@ static void RelayTransaction(const CTransaction& tx, CConnman& connman) static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connman) { - int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) + unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - std::multimap<uint64_t, CNode*> mapMix; const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; - auto sortfunc = [&mapMix, &hasher](CNode* pnode) { + std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; + assert(nRelayNodes <= best.size()); + + auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { if (pnode->nVersion >= CADDR_TIME_VERSION) { uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); - mapMix.emplace(hashKey, pnode); + for (unsigned int i = 0; i < nRelayNodes; i++) { + if (hashKey > best[i].first) { + std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); + best[i] = std::make_pair(hashKey, pnode); + break; + } + } } }; - auto pushfunc = [&addr, &mapMix, &nRelayNodes, &insecure_rand] { - for (auto mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) - mi->second->PushAddress(addr, insecure_rand); + auto pushfunc = [&addr, &best, nRelayNodes, &insecure_rand] { + for (unsigned int i = 0; i < nRelayNodes && best[i].first != 0; i++) { + best[i].second->PushAddress(addr, insecure_rand); + } }; connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); @@ -4828,9 +4887,8 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam { std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); unsigned int nMaxSendBufferSize = connman.GetSendBufferSize(); - vector<CInv> vNotFound; - + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); LOCK(cs_main); while (it != pfrom->vRecvGetData.end()) { @@ -4884,9 +4942,9 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) - pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); else if (inv.type == MSG_WITNESS_BLOCK) - pfrom->PushMessage(NetMsgType::BLOCK, block); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -4899,7 +4957,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } if (sendMerkleBlock) { - pfrom->PushMessage(NetMsgType::MERKLEBLOCK, merkleBlock); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -4908,7 +4966,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, block.vtx[pair.first]); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); } // else // no response @@ -4916,15 +4974,16 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam else if (inv.type == MSG_CMPCT_BLOCK) { // If a peer is asking for old blocks, we're almost guaranteed - // they wont have a useful mempool to match against a compact block, + // they won't have a useful mempool to match against a compact block, // and we don't feel like constructing the object for them, so // instead we respond with the full, non-compact block. bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; + int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - pfrom->PushMessageWithFlag(fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } else - pfrom->PushMessageWithFlag(fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -4935,7 +4994,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // wait for other stuff first. vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - pfrom->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); pfrom->hashContinue.SetNull(); } } @@ -4945,15 +5004,16 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // Send stream from relay memory bool push = false; auto mi = mapRelay.find(inv.hash); + int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); if (mi != mapRelay.end()) { - pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); push = true; } else if (pfrom->timeLastMempoolReq) { auto txinfo = mempool.info(inv.hash); // To protect privacy, do not answer getdata using the mempool when // that TX couldn't have been INVed in reply to a MEMPOOL request. if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); push = true; } } @@ -4980,7 +5040,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // do that because they want to know about (and store and rebroadcast and // risk analyze) the dependencies of transactions relevant to them, without // having to download the entire memory pool. - pfrom->PushMessage(NetMsgType::NOTFOUND, vNotFound); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } @@ -5006,8 +5066,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (!(pfrom->GetLocalServices() & NODE_BLOOM) && (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD || - strCommand == NetMsgType::FILTERCLEAR)) + strCommand == NetMsgType::FILTERADD)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); @@ -5022,16 +5081,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (strCommand == NetMsgType::VERSION) { - // Feeler connections exist only to verify if address is online. - if (pfrom->fFeeler) { - assert(pfrom->fInbound == false); - pfrom->fDisconnect = true; - } - // Each connection can only send one version message if (pfrom->nVersion != 0) { - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message"))); LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; @@ -5051,8 +5104,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pfrom->nServicesExpected & ~pfrom->nServices) { LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; return false; } @@ -5061,8 +5114,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, - strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION)); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; return false; } @@ -5102,7 +5155,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Be shy and don't send version until we hear if (pfrom->fInbound) - pfrom->PushVersion(); + PushNodeVersion(pfrom, connman, GetAdjustedTime()); pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); @@ -5119,8 +5172,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } // Change version - pfrom->PushMessage(NetMsgType::VERACK); - pfrom->ssSend.SetVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + int nSendVersion = std::min(pfrom->nVersion, PROTOCOL_VERSION); + pfrom->SetSendVersion(nSendVersion); if (!pfrom->fInbound) { @@ -5143,7 +5197,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Get recent addresses if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) { - pfrom->PushMessage(NetMsgType::GETADDR); + connman.PushMessage(pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); pfrom->fGetAddr = true; } connman.MarkAddressGood(pfrom->addr); @@ -5163,6 +5217,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeOffset = nTime - GetTime(); pfrom->nTimeOffset = nTimeOffset; AddTimeData(pfrom->addr, nTimeOffset); + + // Feeler connections exist only to verify if address is online. + if (pfrom->fFeeler) { + assert(pfrom->fInbound == false); + pfrom->fDisconnect = true; + } + return true; } @@ -5174,8 +5235,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return false; } + // At this point, the outgoing message serialization version can't change. + CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - else if (strCommand == NetMsgType::VERACK) + if (strCommand == NetMsgType::VERACK) { pfrom->SetRecvVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); @@ -5190,7 +5253,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - pfrom->PushMessage(NetMsgType::SENDHEADERS); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -5201,9 +5264,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom->GetLocalServices() & NODE_WITNESS) - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } } @@ -5323,28 +5386,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom->GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { - // First request the headers preceding the announced block. In the normal fully-synced - // case where a new block is announced that succeeds the current tip (no reorganization), - // there are no such headers. - // Secondly, and only when we are close to being synced, we request the announced block directly, - // to avoid an extra round-trip. Note that we must *first* ask for the headers, so by the - // time the block arrives, the header chain leading up to it is already validated. Not - // doing this will result in the received block being rejected as an orphan in case it is - // not a direct successor. - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash); - CNodeState *nodestate = State(pfrom->GetId()); - if (CanDirectFetch(chainparams.GetConsensus()) && - nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER && - (!IsWitnessEnabled(chainActive.Tip(), chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { - inv.type |= nFetchFlags; - if (nodestate->fSupportsDesiredCmpctVersion) - vToFetch.push_back(CInv(MSG_CMPCT_BLOCK, inv.hash)); - else - vToFetch.push_back(inv); - // Mark block as in flight already, even though the actual "getdata" message only goes out - // later (within the same cs_main lock, though). - MarkBlockAsInFlight(pfrom->GetId(), inv.hash, chainparams.GetConsensus()); - } + // We used to request the full block here, but since headers-announcements are now the + // primary method of announcement on the network, and since, in the case that a node + // fell back to inv we probably have a reorg which we should get the headers for first, + // we now only provide a getheaders response here. When we receive the headers, we will + // then ask for the blocks we need. + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); } } @@ -5367,7 +5414,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } if (!vToFetch.empty()) - pfrom->PushMessage(NetMsgType::GETDATA, vToFetch); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); } @@ -5451,7 +5498,19 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } if (it->second->nHeight < chainActive.Height() - MAX_BLOCKTXN_DEPTH) { + // If an older block is requested (should never happen in practice, + // but can happen in tests) send a block response instead of a + // blocktxn response. Sending a full block response instead of a + // small blocktxn response is preferable in the case where a peer + // might maliciously send lots of getblocktxn requests to trigger + // expensive disk reads, because it will require the peer to + // actually receive all the data read from disk over the network. LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + CInv inv; + inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; + inv.hash = req.blockhash; + pfrom->vRecvGetData.push_back(inv); + ProcessGetData(pfrom, chainparams.GetConsensus(), connman); return true; } @@ -5467,7 +5526,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } resp.txn[i] = block.vtx[req.indexes[i]]; } - pfrom->PushMessageWithFlag(State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCKTXN, resp); + int nSendFlags = State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } @@ -5516,7 +5576,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // headers message). In both cases it's safe to update // pindexBestHeaderSent to be our tip. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); - pfrom->PushMessage(NetMsgType::HEADERS, vHeaders); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); } @@ -5679,13 +5739,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->id, FormatStateMessage(state)); if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); } } - FlushStateToDisk(state, FLUSH_STATE_PERIODIC); } @@ -5699,7 +5758,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!IsInitialBlockDownload()) - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } @@ -5732,7 +5791,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; } @@ -5776,7 +5835,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -5800,7 +5859,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman); } else { req.blockhash = pindex->GetBlockHash(); - pfrom->PushMessage(NetMsgType::GETBLOCKTXN, req); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } } else { @@ -5809,7 +5868,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -5821,8 +5880,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman); } } - - CheckBlockIndex(chainparams.GetConsensus()); } else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing @@ -5853,23 +5910,42 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); - pfrom->PushMessage(NetMsgType::GETDATA, invs); - } else + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + // Block is either okay, or possibly we received + // READ_STATUS_CHECKBLOCK_FAILED. + // Note that CheckBlock can only fail for one of a few reasons: + // 1. bad-proof-of-work (impossible here, because we've already + // accepted the header) + // 2. merkleroot doesn't match the transactions given (already + // caught in FillBlock with READ_STATUS_FAILED, so + // impossible here) + // 3. the block is otherwise invalid (eg invalid coinbase, + // block is too big, too many legacy sigops, etc). + // So if CheckBlock failed, #3 is the only possibility. + // Under BIP 152, we don't DoS-ban unless proof of work is + // invalid (we don't require all the stateless checks to have + // been run). This is handled below, so just treat this as + // though the block was successfully read, and rely on the + // handling in ProcessNewBlock to ensure the block index is + // updated, reject messages go out, etc. + MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer fBlockRead = true; + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + // BIP 152 permits peers to relay compact blocks after validating + // the header only; we should not punish peers if the block turns + // out to be invalid. + mapBlockSource.emplace(resp.blockhash, std::make_pair(pfrom->GetId(), false)); + } } // Don't hold cs_main when we call into ProcessNewBlock if (fBlockRead) { - CValidationState state; - ProcessNewBlock(state, chainparams, pfrom, &block, false, NULL); - int nDoS; - if (state.IsInvalid(nDoS)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - } + bool fNewBlock = false; + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) + ProcessNewBlock(chainparams, &block, true, NULL, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); } } @@ -5911,7 +5987,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // nUnconnectingHeaders gets reset back to 0. if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -5958,7 +6034,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256()); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -6011,12 +6087,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - pfrom->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } - - CheckBlockIndex(chainparams.GetConsensus()); } NotifyHeaderTip(); @@ -6029,24 +6103,25 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); - CValidationState state; // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. // Such an unrequested block may still be processed, subject to the // conditions in AcceptBlock(). bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); - ProcessNewBlock(state, chainparams, pfrom, &block, forceProcessing, NULL); - int nDoS; - if (state.IsInvalid(nDoS)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } + const uint256 hash(block.GetHash()); + { + LOCK(cs_main); + // Also always process if we requested the block explicitly, as we may + // need it even though it is not a candidate for a new best tip. + forceProcessing |= MarkBlockAsReceived(hash); + // mapBlockSource is only used for sending reject messages and DoS scores, + // so the race between here and cs_main in ProcessNewBlock is fine. + mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } - + bool fNewBlock = false; + ProcessNewBlock(chainparams, &block, forceProcessing, NULL, &fNewBlock); + if (fNewBlock) + pfrom->nLastBlockTime = GetTime(); } @@ -6116,7 +6191,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - pfrom->PushMessage(NetMsgType::PONG, nonce); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } } @@ -6228,8 +6303,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (strCommand == NetMsgType::FILTERCLEAR) { LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); + if (pfrom->GetLocalServices() & NODE_BLOOM) { + delete pfrom->pfilter; + pfrom->pfilter = new CBloomFilter(); + } pfrom->fRelayTxes = true; } @@ -6351,7 +6428,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman) // Checksum CDataStream& vRecv = msg.vRecv; - uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); + const uint256& hash = msg.GetMessageHash(); if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, @@ -6370,7 +6447,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman) } catch (const std::ios_base::failure& e) { - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message")); + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message"))); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv @@ -6435,9 +6512,12 @@ bool SendMessages(CNode* pto, CConnman& connman) const Consensus::Params& consensusParams = Params().GetConsensus(); { // Don't send anything until we get its version message - if (pto->nVersion == 0) + if (pto->nVersion == 0 || pto->fDisconnect) return true; + // If we get here, the outgoing message serialization version is set and can't change. + CNetMsgMaker msgMaker(pto->GetSendVersion()); + // // Message: ping // @@ -6450,7 +6530,7 @@ bool SendMessages(CNode* pto, CConnman& connman) // Ping automatically sent as a latency probe & keepalive. pingSend = true; } - if (pingSend && !pto->fDisconnect) { + if (pingSend) { uint64_t nonce = 0; while (nonce == 0) { GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); @@ -6459,11 +6539,11 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - pto->PushMessage(NetMsgType::PING, nonce); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - pto->PushMessage(NetMsgType::PING); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); } } @@ -6471,6 +6551,28 @@ bool SendMessages(CNode* pto, CConnman& connman) if (!lockMain) return true; + CNodeState &state = *State(pto->GetId()); + + BOOST_FOREACH(const CBlockReject& reject, state.rejects) + connman.PushMessage(pto, msgMaker.Make(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock)); + state.rejects.clear(); + + if (state.fShouldBan) { + state.fShouldBan = false; + if (pto->fWhitelisted) + LogPrintf("Warning: not punishing whitelisted peer %s!\n", pto->addr.ToString()); + else { + pto->fDisconnect = true; + if (pto->addr.IsLocal()) + LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); + else + { + connman.Ban(pto->addr, BanReasonNodeMisbehaving); + } + return true; + } + } + // Address refresh broadcast int64_t nNow = GetTimeMicros(); if (!IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { @@ -6494,44 +6596,24 @@ bool SendMessages(CNode* pto, CConnman& connman) // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - pto->PushMessage(NetMsgType::ADDR, vAddr); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - pto->PushMessage(NetMsgType::ADDR, vAddr); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); } - CNodeState &state = *State(pto->GetId()); - if (state.fShouldBan) { - if (pto->fWhitelisted) - LogPrintf("Warning: not punishing whitelisted peer %s!\n", pto->addr.ToString()); - else { - pto->fDisconnect = true; - if (pto->addr.IsLocal()) - LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); - else - { - connman.Ban(pto->addr, BanReasonNodeMisbehaving); - } - } - state.fShouldBan = false; - } - - BOOST_FOREACH(const CBlockReject& reject, state.rejects) - pto->PushMessage(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock); - state.rejects.clear(); - // Start block sync if (pindexBestHeader == NULL) pindexBestHeader = chainActive.Tip(); bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. - if (!state.fSyncStarted && !pto->fClient && !pto->fDisconnect && !fImporting && !fReindex) { + if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; @@ -6547,7 +6629,7 @@ bool SendMessages(CNode* pto, CConnman& connman) if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); - pto->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256()); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -6636,7 +6718,8 @@ bool SendMessages(CNode* pto, CConnman& connman) CBlock block; assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); - pto->PushMessageWithFlag(state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { @@ -6648,7 +6731,7 @@ bool SendMessages(CNode* pto, CConnman& connman) LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); } - pto->PushMessage(NetMsgType::HEADERS, vHeaders); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -6694,7 +6777,7 @@ bool SendMessages(CNode* pto, CConnman& connman) BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -6740,7 +6823,7 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->filterInventoryKnown.insert(hash); vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -6806,7 +6889,7 @@ bool SendMessages(CNode* pto, CConnman& connman) } } if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } pto->filterInventoryKnown.insert(hash); @@ -6814,28 +6897,30 @@ bool SendMessages(CNode* pto, CConnman& connman) } } if (!vInv.empty()) - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling nNow = GetTimeMicros(); - if (!pto->fDisconnect && state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { + if (state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); pto->fDisconnect = true; + return true; } // In case there is a block that has been in flight from this peer for 2 + 0.5 * N times the block interval // (with N the number of peers from which we're downloading validated blocks), disconnect due to timeout. // We compensate for other peers to prevent killing off peers due to our own downstream link // being saturated. We only count validated in-flight blocks so peers can't advertise non-existing block hashes // to unreasonably increase our timeout. - if (!pto->fDisconnect && state.vBlocksInFlight.size() > 0) { + if (state.vBlocksInFlight.size() > 0) { QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); pto->fDisconnect = true; + return true; } } @@ -6843,7 +6928,7 @@ bool SendMessages(CNode* pto, CConnman& connman) // Message: getdata (blocks) // vector<CInv> vGetData; - if (!pto->fDisconnect && !pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { vector<CBlockIndex*> vToDownload; NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); @@ -6865,7 +6950,7 @@ bool SendMessages(CNode* pto, CConnman& connman) // // Message: getdata (non-blocks) // - while (!pto->fDisconnect && !pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) + while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) { const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) @@ -6875,7 +6960,7 @@ bool SendMessages(CNode* pto, CConnman& connman) vGetData.push_back(inv); if (vGetData.size() >= 1000) { - pto->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); vGetData.clear(); } } else { @@ -6885,7 +6970,7 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->mapAskFor.erase(pto->mapAskFor.begin()); } if (!vGetData.empty()) - pto->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); // // Message: feefilter @@ -6898,7 +6983,7 @@ bool SendMessages(CNode* pto, CConnman& connman) if (timeNow > pto->nextSendTimeFeeFilter) { CAmount filterToSend = filterRounder.round(currentFilter); if (filterToSend != pto->lastSentFeeFilter) { - pto->PushMessage(NetMsgType::FEEFILTER, filterToSend); + connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); pto->lastSentFeeFilter = filterToSend; } pto->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); @@ -6930,6 +7015,119 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); } +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(void) +{ + int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "r"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t skipped = 0; + int64_t failed = 0; + int64_t nNow = GetTime(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + double prioritydummy = 0; + while (num--) { + CTransaction tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + mempool.PrioritiseTransaction(tx.GetHash(), tx.GetHash().ToString(), prioritydummy, amountdelta); + } + CValidationState state; + if (nTime + nExpiryTimeout > nNow) { + LOCK(cs_main); + AcceptToMemoryPoolWithTime(mempool, state, tx, true, NULL, nTime); + if (state.IsValid()) { + ++count; + } else { + ++failed; + } + } else { + ++skipped; + } + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + return true; +} + +void DumpMempool(void) +{ + int64_t start = GetTimeMicros(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + + { + LOCK(mempool.cs); + for (const auto &i : mempool.mapDeltas) { + mapDeltas[i.first] = i.second.first; + } + vinfo = mempool.infoAll(); + } + + int64_t mid = GetTimeMicros(); + + try { + FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "w"); + if (!filestr) { + return; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << (int64_t)i.nTime; + file << (int64_t)i.nFeeDelta; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + FileCommit(file.Get()); + file.fclose(); + RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + int64_t last = GetTimeMicros(); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*0.000001, (last-mid)*0.000001); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + } +} + class CMainCleanup { public: diff --git a/src/main.h b/src/main.h index 3eab9b89da..c98ed05726 100644 --- a/src/main.h +++ b/src/main.h @@ -125,7 +125,7 @@ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; /** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; -static const unsigned int DEFAULT_LIMITFREERELAY = 15; +static const unsigned int DEFAULT_LIMITFREERELAY = 0; static const bool DEFAULT_RELAYPRIORITY = true; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; @@ -165,7 +165,7 @@ extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; extern CWaitableCriticalSection csBestBlock; extern CConditionVariable cvBlockChange; -extern bool fImporting; +extern std::atomic_bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; @@ -215,15 +215,21 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the * specific block passed to it has been checked for validity! + * + * If you want to *possibly* get feedback on whether pblock is valid, you must + * install a CValidationInterface (see validationinterface.h) - this will have + * its BlockChecked method called whenever *any* block completes validation. + * + * Note that we guarantee that either the proof-of-work is valid on pblock, or + * (and possibly also) BlockChecked will have been called. * - * @param[out] state This may be set to an Error state if any error occurred processing it, including during validation/connection/etc of otherwise unrelated blocks during reorganization; or it may be set to an Invalid state if pblock is itself invalid (but this is not guaranteed even when the block is checked). If you want to *possibly* get feedback on whether pblock is valid, you must also install a CValidationInterface (see validationinterface.h) - this will have its BlockChecked method called whenever *any* block completes validation. - * @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be penalised if the block is invalid. * @param[in] pblock The block we want to process. * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. * @param[out] dbp The already known disk position of pblock, or NULL if not yet stored. + * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call * @return True if state.IsValid() */ -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp); +bool ProcessNewBlock(const CChainParams& chainparams, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp, bool* fNewBlock); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); /** Open a block file (blk?????.dat) */ @@ -237,7 +243,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB /** Initialize a new block tree database + block data on disk */ bool InitBlockIndex(const CChainParams& chainparams); /** Load the block tree and coins database from disk */ -bool LoadBlockIndex(); +bool LoadBlockIndex(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ @@ -291,6 +297,10 @@ void PruneAndFlush(); bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); +/** (try to) add transaction to memory pool with a specified acceptance time **/ +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); + /** Convert CValidationState to a human-readable message for logging */ std::string FormatStateMessage(const CValidationState &state); @@ -339,7 +349,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); /** Transaction validation functions */ /** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state); +bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); namespace Consensus { @@ -529,6 +539,12 @@ static const unsigned int REJECT_ALREADY_KNOWN = 0x101; /** Transaction conflicts with a transaction already known */ static const unsigned int REJECT_CONFLICT = 0x102; +/** Dump the mempool to disk. */ +void DumpMempool(); + +/** Load the mempool from disk. */ +bool LoadMempool(); + // The following things handle network-processing logic // (and should be moved to a separate file) @@ -542,8 +558,9 @@ private: CConnman* connman; public: - PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) {} + PeerLogicValidation(CConnman* connmanIn); + virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); virtual void BlockChecked(const CBlock& block, const CValidationState& state); }; diff --git a/src/memusage.h b/src/memusage.h index 3810bfad07..2e3c5a9b92 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin developers +// Copyright (c) 2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 31332526a9..882717ac56 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -23,8 +23,8 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter& filter) for (unsigned int i = 0; i < block.vtx.size(); i++) { - const uint256& hash = block.vtx[i].GetHash(); - if (filter.IsRelevantAndUpdate(block.vtx[i])) + const uint256& hash = block.vtx[i]->GetHash(); + if (filter.IsRelevantAndUpdate(*block.vtx[i])) { vMatch.push_back(true); vMatchedTxn.push_back(make_pair(i, hash)); @@ -49,7 +49,7 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, const std::set<uint256>& txids) for (unsigned int i = 0; i < block.vtx.size(); i++) { - const uint256& hash = block.vtx[i].GetHash(); + const uint256& hash = block.vtx[i]->GetHash(); if (txids.count(hash)) vMatch.push_back(true); else diff --git a/src/merkleblock.h b/src/merkleblock.h index 835cbcce55..17c33194a9 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -85,7 +85,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nTransactions); READWRITE(vHash); std::vector<unsigned char> vBytes; @@ -148,7 +148,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(header); READWRITE(txn); } diff --git a/src/miner.cpp b/src/miner.cpp index ebf2f21ffd..c40b12cd8e 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -134,7 +134,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock = &pblocktemplate->block; // pointer for convenience // Add dummy coinbase tx as first transaction - pblock->vtx.push_back(CTransaction()); + pblock->vtx.emplace_back(); pblocktemplate->vTxFees.push_back(-1); // updated at end pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end @@ -169,7 +169,6 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc nLastBlockTx = nBlockTx; nLastBlockSize = nBlockSize; nLastBlockWeight = nBlockWeight; - LogPrintf("CreateNewBlock(): total size %u txs: %u fees: %ld sigops %d\n", nBlockSize, nBlockTx, nFees, nBlockSigOpsCost); // Create coinbase transaction. CMutableTransaction coinbaseTx; @@ -179,16 +178,19 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc coinbaseTx.vout[0].scriptPubKey = scriptPubKeyIn; coinbaseTx.vout[0].nValue = nFees + GetBlockSubsidy(nHeight, chainparams.GetConsensus()); coinbaseTx.vin[0].scriptSig = CScript() << nHeight << OP_0; - pblock->vtx[0] = coinbaseTx; + pblock->vtx[0] = MakeTransactionRef(std::move(coinbaseTx)); pblocktemplate->vchCoinbaseCommitment = GenerateCoinbaseCommitment(*pblock, pindexPrev, chainparams.GetConsensus()); pblocktemplate->vTxFees[0] = -nFees; + uint64_t nSerializeSize = GetSerializeSize(*pblock, SER_NETWORK, PROTOCOL_VERSION); + LogPrintf("CreateNewBlock(): total size: %u block weight: %u txs: %u fees: %ld sigops %d\n", nSerializeSize, GetBlockWeight(*pblock), nBlockTx, nFees, nBlockSigOpsCost); + // Fill in header pblock->hashPrevBlock = pindexPrev->GetBlockHash(); UpdateTime(pblock, chainparams.GetConsensus(), pindexPrev); pblock->nBits = GetNextWorkRequired(pindexPrev, pblock, chainparams.GetConsensus()); pblock->nNonce = 0; - pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(pblock->vtx[0]); + pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); CValidationState state; if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { @@ -310,7 +312,7 @@ bool BlockAssembler::TestForBlock(CTxMemPool::txiter iter) void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) { - pblock->vtx.push_back(iter->GetTx()); + pblock->vtx.emplace_back(iter->GetSharedTx()); pblocktemplate->vTxFees.push_back(iter->GetFee()); pblocktemplate->vTxSigOpsCost.push_back(iter->GetSigOpCost()); if (fNeedSizeAccounting) { @@ -599,10 +601,10 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned } ++nExtraNonce; unsigned int nHeight = pindexPrev->nHeight+1; // Height first in coinbase required for block.version=2 - CMutableTransaction txCoinbase(pblock->vtx[0]); + CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)) + COINBASE_FLAGS; assert(txCoinbase.vin[0].scriptSig.size() <= 100); - pblock->vtx[0] = txCoinbase; + pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); } diff --git a/src/net.cpp b/src/net.cpp index 1bca168d1d..ba74cd071e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -64,6 +64,7 @@ const static std::string NET_MESSAGE_COMMAND_OTHER = "*other*"; static const uint64_t RANDOMIZER_ID_NETGROUP = 0x6c0edd8036ef4036ULL; // SHA256("netgroup")[0:8] +static const uint64_t RANDOMIZER_ID_LOCALHOSTNONCE = 0xd93e69e2bbfa5735ULL; // SHA256("localhostnonce")[0:8] // // Global state variables // @@ -389,18 +390,18 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addrman.Attempt(addrConnect, fCountFailure); // Add node - CNode* pnode = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), pszDest ? pszDest : "", false); - GetNodeSignals().InitializeNode(pnode->GetId(), pnode); + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, pszDest ? pszDest : "", false); + pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); + pnode->nTimeConnected = GetTime(); pnode->AddRef(); - + GetNodeSignals().InitializeNode(pnode, *this); { LOCK(cs_vNodes); vNodes.push_back(pnode); } - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); - pnode->nTimeConnected = GetTime(); - return pnode; } else if (!proxyConnectionFailed) { // If connecting to the node failed, and failure is not caused by a problem connecting to @@ -446,23 +447,6 @@ void CNode::CloseSocketDisconnect() vRecvMsg.clear(); } -void CNode::PushVersion() -{ - int64_t nTime = (fInbound ? GetAdjustedTime() : GetTime()); - CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); - CAddress addrMe = CAddress(CService(), nLocalServices); - if (fLogIPs) - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nMyStartingHeight, addrMe.ToString(), addrYou.ToString(), id); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nMyStartingHeight, addrMe.ToString(), id); - PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalServices, nTime, addrYou, addrMe, - nLocalHostNonce, strSubVersion, nMyStartingHeight, ::fRelayTxes); -} - - - - - void CConnman::ClearBanned() { { @@ -758,12 +742,21 @@ int CNetMessage::readData(const char *pch, unsigned int nBytes) vRecv.resize(std::min(hdr.nMessageSize, nDataPos + nCopy + 256 * 1024)); } + hasher.Write((const unsigned char*)pch, nCopy); memcpy(&vRecv[nDataPos], pch, nCopy); nDataPos += nCopy; return nCopy; } +const uint256& CNetMessage::GetMessageHash() const +{ + assert(complete()); + if (data_hash.IsNull()) + hasher.Finalize(data_hash.begin()); + return data_hash; +} + @@ -775,13 +768,13 @@ int CNetMessage::readData(const char *pch, unsigned int nBytes) // requires LOCK(cs_vSend) size_t SocketSendData(CNode *pnode) { - std::deque<CSerializeData>::iterator it = pnode->vSendMsg.begin(); + auto it = pnode->vSendMsg.begin(); size_t nSentSize = 0; while (it != pnode->vSendMsg.end()) { - const CSerializeData &data = *it; + const auto &data = *it; assert(data.size() > pnode->nSendOffset); - int nBytes = send(pnode->hSocket, &data[pnode->nSendOffset], data.size() - pnode->nSendOffset, MSG_NOSIGNAL | MSG_DONTWAIT); + int nBytes = send(pnode->hSocket, reinterpret_cast<const char*>(data.data()) + pnode->nSendOffset, data.size() - pnode->nSendOffset, MSG_NOSIGNAL | MSG_DONTWAIT); if (nBytes > 0) { pnode->nLastSend = GetTime(); pnode->nSendBytes += nBytes; @@ -825,7 +818,7 @@ struct NodeEvictionCandidate int64_t nMinPingUsecTime; int64_t nLastBlockTime; int64_t nLastTXTime; - bool fNetworkNode; + bool fRelevantServices; bool fRelayTxes; bool fBloomFilter; CAddress addr; @@ -850,7 +843,7 @@ static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvict { // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. if (a.nLastBlockTime != b.nLastBlockTime) return a.nLastBlockTime < b.nLastBlockTime; - if (a.fNetworkNode != b.fNetworkNode) return b.fNetworkNode; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; return a.nTimeConnected > b.nTimeConnected; } @@ -885,7 +878,8 @@ bool CConnman::AttemptToEvictConnection() if (node->fDisconnect) continue; NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, - node->nLastBlockTime, node->nLastTXTime, node->fNetworkNode, + node->nLastBlockTime, node->nLastTXTime, + (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } @@ -970,7 +964,6 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { CAddress addr; int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - assert(nMaxInbound > 0); if (hSocket != INVALID_SOCKET) if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) @@ -992,6 +985,12 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { return; } + if (!fNetworkActive) { + LogPrintf("connection from %s dropped: not accepting new connections\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + if (!IsSelectableSocket(hSocket)) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); @@ -1025,10 +1024,13 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } } - CNode* pnode = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), "", true); - GetNodeSignals().InitializeNode(pnode->GetId(), pnode); + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; + GetNodeSignals().InitializeNode(pnode, *this); LogPrint("net", "connection from %s accepted\n", addr.ToString()); @@ -1053,7 +1055,7 @@ void CConnman::ThreadSocketHandler() BOOST_FOREACH(CNode* pnode, vNodesCopy) { if (pnode->fDisconnect || - (pnode->GetRefCount() <= 0 && pnode->vRecvMsg.empty() && pnode->nSendSize == 0 && pnode->ssSend.empty())) + (pnode->GetRefCount() <= 0 && pnode->vRecvMsg.empty() && pnode->nSendSize == 0)) { // remove from vNodes vNodes.erase(remove(vNodes.begin(), vNodes.end(), pnode), vNodes.end()); @@ -1101,8 +1103,13 @@ void CConnman::ThreadSocketHandler() } } } - if(vNodes.size() != nPrevNodeCount) { - nPrevNodeCount = vNodes.size(); + size_t vNodesSize; + { + LOCK(cs_vNodes); + vNodesSize = vNodes.size(); + } + if(vNodesSize != nPrevNodeCount) { + nPrevNodeCount = vNodesSize; if(clientInterface) clientInterface->NotifyNumConnectionsChanged(nPrevNodeCount); } @@ -1157,10 +1164,6 @@ void CConnman::ThreadSocketHandler() { TRY_LOCK(pnode->cs_vSend, lockSend); if (lockSend) { - if (pnode->nOptimisticBytesWritten) { - RecordBytesSent(pnode->nOptimisticBytesWritten); - pnode->nOptimisticBytesWritten = 0; - } if (!pnode->vSendMsg.empty()) { FD_SET(pnode->hSocket, &fdsetSend); continue; @@ -1792,6 +1795,9 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai // Initiate outbound network connection // boost::this_thread::interruption_point(); + if (!fNetworkActive) { + return false; + } if (!pszDest) { if (IsLocal(addrConnect) || FindNode((CNetAddr)addrConnect) || IsBanned(addrConnect) || @@ -2033,8 +2039,30 @@ void Discover(boost::thread_group& threadGroup) #endif } +void CConnman::SetNetworkActive(bool active) +{ + if (fDebug) { + LogPrint("net", "SetNetworkActive: %s\n", active); + } + + if (!active) { + fNetworkActive = false; + + LOCK(cs_vNodes); + // Close sockets to all nodes + BOOST_FOREACH(CNode* pnode, vNodes) { + pnode->CloseSocketDisconnect(); + } + } else { + fNetworkActive = true; + } + + uiInterface.NotifyNetworkActiveChanged(fNetworkActive); +} + CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSeed1(nSeed1In) { + fNetworkActive = true; setBannedIsDirty = false; fAddressesInitialized = false; nLastNodeId = 0; @@ -2119,8 +2147,12 @@ bool CConnman::Start(boost::thread_group& threadGroup, CScheduler& scheduler, st if (pnodeLocalHost == NULL) { CNetAddr local; LookupHost("127.0.0.1", local, false); - pnodeLocalHost = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), INVALID_SOCKET, CAddress(CService(local, 0), nLocalServices), 0); - GetNodeSignals().InitializeNode(pnodeLocalHost->GetId(), pnodeLocalHost); + + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + + pnodeLocalHost = new CNode(id, nLocalServices, GetBestHeight(), INVALID_SOCKET, CAddress(CService(local, 0), nLocalServices), 0, nonce); + GetNodeSignals().InitializeNode(pnodeLocalHost, *this); } // @@ -2138,8 +2170,9 @@ bool CConnman::Start(boost::thread_group& threadGroup, CScheduler& scheduler, st // Initiate outbound connections from -addnode threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "addcon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenAddedConnections, this)))); - // Initiate outbound connections - threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); + // Initiate outbound connections unless connect=0 + if (!mapArgs.count("-connect") || mapMultiArgs["-connect"].size() != 1 || mapMultiArgs["-connect"][0] != "0") + threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); // Process messages threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "msghand", boost::function<void()>(boost::bind(&CConnman::ThreadMessageHandler, this)))); @@ -2471,50 +2504,20 @@ int CConnman::GetBestHeight() const return nBestHeight.load(std::memory_order_acquire); } -void CNode::Fuzz(int nChance) -{ - if (!fSuccessfullyConnected) return; // Don't fuzz initial handshake - if (GetRand(nChance) != 0) return; // Fuzz 1 of every nChance messages - - switch (GetRand(3)) - { - case 0: - // xor a random byte with a random value: - if (!ssSend.empty()) { - CDataStream::size_type pos = GetRand(ssSend.size()); - ssSend[pos] ^= (unsigned char)(GetRand(256)); - } - break; - case 1: - // delete a random byte: - if (!ssSend.empty()) { - CDataStream::size_type pos = GetRand(ssSend.size()); - ssSend.erase(ssSend.begin()+pos); - } - break; - case 2: - // insert a random byte at a random position - { - CDataStream::size_type pos = GetRand(ssSend.size()); - char ch = (char)GetRand(256); - ssSend.insert(ssSend.begin()+pos, ch); - } - break; - } - // Chance of more than one change half the time: - // (more changes exponentially less likely): - Fuzz(2); -} - unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, const std::string& addrNameIn, bool fInboundIn) : - ssSend(SER_NETWORK, INIT_PROTO_VERSION), +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string& addrNameIn, bool fInboundIn) : addr(addrIn), + fInbound(fInboundIn), + id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), - filterInventoryKnown(50000, 0.000001) + filterInventoryKnown(50000, 0.000001), + nLocalHostNonce(nLocalHostNonceIn), + nLocalServices(nLocalServicesIn), + nMyStartingHeight(nMyStartingHeightIn), + nSendVersion(0) { nServices = NODE_NONE; nServicesExpected = NODE_NONE; @@ -2533,7 +2536,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn fOneShot = false; fClient = false; // set by version message fFeeler = false; - fInbound = fInboundIn; fNetworkNode = false; fSuccessfullyConnected = false; fDisconnect = false; @@ -2562,12 +2564,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn minFeeFilter = 0; lastSentFeeFilter = 0; nextSendTimeFeeFilter = 0; - id = idIn; - nOptimisticBytesWritten = 0; - nLocalServices = nLocalServicesIn; - - GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); - nMyStartingHeight = nMyStartingHeightIn; BOOST_FOREACH(const std::string &msg, getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; @@ -2577,10 +2573,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn LogPrint("net", "Added connection to %s peer=%d\n", addrName, id); else LogPrint("net", "Added connection peer=%d\n", id); - - // Be shy and don't send version until we hear - if (hSocket != INVALID_SOCKET && !fInbound) - PushVersion(); } CNode::~CNode() @@ -2625,65 +2617,42 @@ void CNode::AskFor(const CInv& inv) mapAskFor.insert(std::make_pair(nRequestTime, inv)); } -void CNode::BeginMessage(const char* pszCommand) EXCLUSIVE_LOCK_FUNCTION(cs_vSend) +void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { - ENTER_CRITICAL_SECTION(cs_vSend); - assert(ssSend.size() == 0); - ssSend << CMessageHeader(Params().MessageStart(), pszCommand, 0); - LogPrint("net", "sending: %s ", SanitizeString(pszCommand)); -} + size_t nMessageSize = msg.data.size(); + size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; + LogPrint("net", "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); -void CNode::AbortMessage() UNLOCK_FUNCTION(cs_vSend) -{ - ssSend.clear(); + std::vector<unsigned char> serializedHeader; + serializedHeader.reserve(CMessageHeader::HEADER_SIZE); + uint256 hash = Hash(msg.data.data(), msg.data.data() + nMessageSize); + CMessageHeader hdr(Params().MessageStart(), msg.command.c_str(), nMessageSize); + memcpy(hdr.pchChecksum, hash.begin(), CMessageHeader::CHECKSUM_SIZE); - LEAVE_CRITICAL_SECTION(cs_vSend); - - LogPrint("net", "(aborted)\n"); -} - -void CNode::EndMessage(const char* pszCommand) UNLOCK_FUNCTION(cs_vSend) -{ - // The -*messagestest options are intentionally not documented in the help message, - // since they are only used during development to debug the networking code and are - // not intended for end-users. - if (mapArgs.count("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 2)) == 0) - { - LogPrint("net", "dropmessages DROPPING SEND MESSAGE\n"); - AbortMessage(); - return; - } - if (mapArgs.count("-fuzzmessagestest")) - Fuzz(GetArg("-fuzzmessagestest", 10)); + CVectorWriter{SER_NETWORK, INIT_PROTO_VERSION, serializedHeader, 0, hdr}; - if (ssSend.size() == 0) + size_t nBytesSent = 0; { - LEAVE_CRITICAL_SECTION(cs_vSend); - return; - } - // Set the size - unsigned int nSize = ssSend.size() - CMessageHeader::HEADER_SIZE; - WriteLE32((uint8_t*)&ssSend[CMessageHeader::MESSAGE_SIZE_OFFSET], nSize); - - //log total amount of bytes per command - mapSendBytesPerMsgCmd[std::string(pszCommand)] += nSize + CMessageHeader::HEADER_SIZE; - - // Set the checksum - uint256 hash = Hash(ssSend.begin() + CMessageHeader::HEADER_SIZE, ssSend.end()); - assert(ssSend.size () >= CMessageHeader::CHECKSUM_OFFSET + CMessageHeader::CHECKSUM_SIZE); - memcpy((char*)&ssSend[CMessageHeader::CHECKSUM_OFFSET], hash.begin(), CMessageHeader::CHECKSUM_SIZE); - - LogPrint("net", "(%d bytes) peer=%d\n", nSize, id); + LOCK(pnode->cs_vSend); + if(pnode->hSocket == INVALID_SOCKET) { + return; + } + bool optimisticSend(pnode->vSendMsg.empty()); - std::deque<CSerializeData>::iterator it = vSendMsg.insert(vSendMsg.end(), CSerializeData()); - ssSend.GetAndClear(*it); - nSendSize += (*it).size(); + //log total amount of bytes per command + pnode->mapSendBytesPerMsgCmd[msg.command] += nTotalSize; + pnode->nSendSize += nTotalSize; - // If write queue empty, attempt "optimistic write" - if (it == vSendMsg.begin()) - nOptimisticBytesWritten += SocketSendData(this); + pnode->vSendMsg.push_back(std::move(serializedHeader)); + if (nMessageSize) + pnode->vSendMsg.push_back(std::move(msg.data)); - LEAVE_CRITICAL_SECTION(cs_vSend); + // If write queue empty, attempt "optimistic write" + if (optimisticSend == true) + nBytesSent = SocketSendData(pnode); + } + if (nBytesSent) + RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -101,6 +101,20 @@ class CTransaction; class CNodeStats; class CClientUIInterface; +struct CSerializedNetMsg +{ + CSerializedNetMsg() = default; + CSerializedNetMsg(CSerializedNetMsg&&) = default; + CSerializedNetMsg& operator=(CSerializedNetMsg&&) = default; + // No copying, only moves. + CSerializedNetMsg(const CSerializedNetMsg& msg) = delete; + CSerializedNetMsg& operator=(const CSerializedNetMsg&) = delete; + + std::vector<unsigned char> data; + std::string command; +}; + + class CConnman { public: @@ -131,11 +145,15 @@ public: bool Start(boost::thread_group& threadGroup, CScheduler& scheduler, std::string& strNodeError, Options options); void Stop(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); + bool GetNetworkActive() const { return fNetworkActive; }; + void SetNetworkActive(bool active); bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = NULL, const char *strDest = NULL, bool fOneShot = false, bool fFeeler = false); bool CheckIncomingNonce(uint64_t nonce); bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); + void PushMessage(CNode* pnode, CSerializedNetMsg&& msg); + template<typename Callable> bool ForEachNodeContinueIf(Callable&& func) { @@ -370,6 +388,7 @@ private: unsigned int nReceiveFloodSize; std::vector<ListenSocket> vhListenSocket; + std::atomic<bool> fNetworkActive; banmap_t setBanned; CCriticalSection cs_setBanned; bool setBannedIsDirty; @@ -428,7 +447,7 @@ struct CNodeSignals { boost::signals2::signal<bool (CNode*, CConnman&), CombinerAll> ProcessMessages; boost::signals2::signal<bool (CNode*, CConnman&), CombinerAll> SendMessages; - boost::signals2::signal<void (NodeId, const CNode*)> InitializeNode; + boost::signals2::signal<void (CNode*, CConnman&)> InitializeNode; boost::signals2::signal<void (NodeId, bool&)> FinalizeNode; }; @@ -512,6 +531,9 @@ public: class CNetMessage { +private: + mutable CHash256 hasher; + mutable uint256 data_hash; public: bool in_data; // parsing header (false) or data (true) @@ -539,6 +561,8 @@ public: return (hdr.nMessageSize == nDataPos); } + const uint256& GetMessageHash() const; + void SetVersion(int nVersionIn) { hdrbuf.SetVersion(nVersionIn); @@ -553,17 +577,16 @@ public: /** Information about a peer */ class CNode { + friend class CConnman; public: // socket ServiceFlags nServices; ServiceFlags nServicesExpected; SOCKET hSocket; - CDataStream ssSend; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent - uint64_t nOptimisticBytesWritten; uint64_t nSendBytes; - std::deque<CSerializeData> vSendMsg; + std::deque<std::vector<unsigned char>> vSendMsg; CCriticalSection cs_vSend; std::deque<CInv> vRecvGetData; @@ -589,10 +612,10 @@ public: bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; bool fClient; - bool fInbound; + const bool fInbound; bool fNetworkNode; bool fSuccessfullyConnected; - bool fDisconnect; + std::atomic_bool fDisconnect; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs @@ -603,7 +626,7 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; int nRefCount; - NodeId id; + const NodeId id; const uint64_t nKeyedNetGroup; protected: @@ -611,9 +634,6 @@ protected: mapMsgCmdSize mapSendBytesPerMsgCmd; mapMsgCmdSize mapRecvBytesPerMsgCmd; - // Basic fuzz-testing - void Fuzz(int nChance); // modifies ssSend - public: uint256 hashContinue; int nStartingHeight; @@ -669,7 +689,7 @@ public: CAmount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, const std::string &addrNameIn = "", bool fInboundIn = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: @@ -677,10 +697,11 @@ private: void operator=(const CNode&); - uint64_t nLocalHostNonce; + const uint64_t nLocalHostNonce; // Services offered to this peer - ServiceFlags nLocalServices; - int nMyStartingHeight; + const ServiceFlags nLocalServices; + const int nMyStartingHeight; + int nSendVersion; public: NodeId GetId() const { @@ -691,6 +712,10 @@ public: return nLocalHostNonce; } + int GetMyStartingHeight() const { + return nMyStartingHeight; + } + int GetRefCount() { assert(nRefCount >= 0); @@ -716,6 +741,25 @@ public: BOOST_FOREACH(CNetMessage &msg, vRecvMsg) msg.SetVersion(nVersionIn); } + void SetSendVersion(int nVersionIn) + { + // Send version may only be changed in the version message, and + // only one version message is allowed per session. We can therefore + // treat this value as const and even atomic as long as it's only used + // once the handshake is complete. Any attempt to set this twice is an + // error. + assert(nSendVersion == 0); + nSendVersion = nVersionIn; + } + + int GetSendVersion() const + { + // The send version should always be explicitly set to + // INIT_PROTO_VERSION rather than using this value until the handshake + // is complete. + assert(nSendVersion != 0); + return nSendVersion; + } CNode* AddRef() { @@ -778,193 +822,6 @@ public: void AskFor(const CInv& inv); - // TODO: Document the postcondition of this function. Is cs_vSend locked? - void BeginMessage(const char* pszCommand) EXCLUSIVE_LOCK_FUNCTION(cs_vSend); - - // TODO: Document the precondition of this function. Is cs_vSend locked? - void AbortMessage() UNLOCK_FUNCTION(cs_vSend); - - // TODO: Document the precondition of this function. Is cs_vSend locked? - void EndMessage(const char* pszCommand) UNLOCK_FUNCTION(cs_vSend); - - void PushVersion(); - - - void PushMessage(const char* pszCommand) - { - try - { - BeginMessage(pszCommand); - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1> - void PushMessage(const char* pszCommand, const T1& a1) - { - try - { - BeginMessage(pszCommand); - ssSend << a1; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - /** Send a message containing a1, serialized with flag flag. */ - template<typename T1> - void PushMessageWithFlag(int flag, const char* pszCommand, const T1& a1) - { - try - { - BeginMessage(pszCommand); - WithOrVersion(&ssSend, flag) << a1; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7, const T8& a8) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7 << a8; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7, const T8& a8, const T9& a9) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7 << a8 << a9; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats); diff --git a/src/netaddress.h b/src/netaddress.h index 9330fe3328..9dffaa57e7 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -85,7 +85,7 @@ class CNetAddr ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(FLATDATA(ip)); } @@ -122,7 +122,7 @@ class CSubNet ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(network); READWRITE(FLATDATA(netmask)); READWRITE(FLATDATA(valid)); @@ -159,7 +159,7 @@ class CService : public CNetAddr ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(FLATDATA(ip)); unsigned short portN = htons(port); READWRITE(FLATDATA(portN)); diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h new file mode 100644 index 0000000000..7167434a19 --- /dev/null +++ b/src/netmessagemaker.h @@ -0,0 +1,36 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NETMESSAGEMAKER_H +#define BITCOIN_NETMESSAGEMAKER_H + +#include "net.h" +#include "serialize.h" + +class CNetMsgMaker +{ +public: + CNetMsgMaker(int nVersionIn) : nVersion(nVersionIn){} + + template <typename... Args> + CSerializedNetMsg Make(int nFlags, std::string sCommand, Args&&... args) + { + CSerializedNetMsg msg; + msg.command = std::move(sCommand); + CVectorWriter{ SER_NETWORK, nFlags | nVersion, msg.data, 0, std::forward<Args>(args)... }; + return msg; + } + + template <typename... Args> + CSerializedNetMsg Make(std::string sCommand, Args&&... args) + { + return Make(0, std::move(sCommand), std::forward<Args>(args)...); + } + +private: + const int nVersion; +}; + +#endif // BITCOIN_NETMESSAGEMAKER_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index c07cd2eff8..7113390cdf 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,10 +14,9 @@ #include "util.h" void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay, std::string _dataTypeString) + unsigned int maxConfirms, double _decay) { decay = _decay; - dataTypeString = _dataTypeString; for (unsigned int i = 0; i < defaultBuckets.size(); i++) { buckets.push_back(defaultBuckets[i]); bucketMap[defaultBuckets[i]] = i; @@ -87,10 +86,10 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, int maxbucketindex = buckets.size() - 1; - // requireGreater means we are looking for the lowest fee/priority such that all higher - // values pass, so we start at maxbucketindex (highest fee) and look at successively + // requireGreater means we are looking for the lowest feerate such that all higher + // values pass, so we start at maxbucketindex (highest feerate) and look at successively // smaller buckets until we reach failure. Otherwise, we are looking for the highest - // fee/priority such that all lower values fail, and we go in the opposite direction. + // feerate such that all lower values fail, and we go in the opposite direction. unsigned int startbucket = requireGreater ? maxbucketindex : 0; int step = requireGreater ? -1 : 1; @@ -107,7 +106,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); - // Start counting from highest(default) or lowest fee/pri transactions + // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { curFarBucket = bucket; nConf += confAvg[confTarget - 1][bucket]; @@ -145,8 +144,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double median = -1; double txSum = 0; - // Calculate the "average" fee of the best bucket range that met success conditions - // Find the bucket with the median transaction and then report the average fee from that bucket + // Calculate the "average" feerate of the best bucket range that met success conditions + // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; @@ -166,8 +165,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, } } - LogPrint("estimatefee", "%3d: For conf success %s %4.2f need %s %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, dataTypeString, + LogPrint("estimatefee", "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", + confTarget, requireGreater ? ">" : "<", successBreakPoint, requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); @@ -200,10 +199,10 @@ void TxConfirmStats::Read(CAutoFile& filein) filein >> fileBuckets; numBuckets = fileBuckets.size(); if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 fee/pri buckets"); + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); filein >> fileAvg; if (fileAvg.size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri average bucket count"); + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); filein >> fileTxCtAvg; if (fileTxCtAvg.size() != numBuckets) throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); @@ -213,9 +212,9 @@ void TxConfirmStats::Read(CAutoFile& filein) throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); for (unsigned int i = 0; i < maxConfirms; i++) { if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri conf average bucket count"); + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire fee estimate data file and not + // Now that we've processed the entire feerate estimate data file and not // thrown any errors, we can copy it to our data structures decay = fileDecay; buckets = fileBuckets; @@ -242,8 +241,8 @@ void TxConfirmStats::Read(CAutoFile& filein) for (unsigned int i = 0; i < buckets.size(); i++) bucketMap[buckets[i]] = i; - LogPrint("estimatefee", "Reading estimates: %u %s buckets counting confirms up to %u blocks\n", - numBuckets, dataTypeString, maxConfirms); + LogPrint("estimatefee", "Reading estimates: %u buckets counting confirms up to %u blocks\n", + numBuckets, maxConfirms); } unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) @@ -251,7 +250,6 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) unsigned int bucketindex = bucketMap.lower_bound(val)->second; unsigned int blockIndex = nBlockHeight % unconfTxs.size(); unconfTxs[blockIndex][bucketindex]++; - LogPrint("estimatefee", "adding to %s", dataTypeString); return bucketindex; } @@ -291,12 +289,10 @@ void CBlockPolicyEstimator::removeTx(uint256 hash) hash.ToString().c_str()); return; } - TxConfirmStats *stats = pos->second.stats; unsigned int entryHeight = pos->second.blockHeight; unsigned int bucketIndex = pos->second.bucketIndex; - if (stats != NULL) - stats->removeTx(entryHeight, nBestSeenHeight, bucketIndex); + feeStats.removeTx(entryHeight, nBestSeenHeight, bucketIndex); mapMemPoolTxs.erase(hash); } @@ -309,45 +305,14 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate& _minRelayFee) vfeelist.push_back(bucketBoundary); } vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "FeeRate"); - - minTrackedPriority = AllowFreeThreshold() < MIN_PRIORITY ? MIN_PRIORITY : AllowFreeThreshold(); - std::vector<double> vprilist; - for (double bucketBoundary = minTrackedPriority; bucketBoundary <= MAX_PRIORITY; bucketBoundary *= PRI_SPACING) { - vprilist.push_back(bucketBoundary); - } - vprilist.push_back(INF_PRIORITY); - priStats.Initialize(vprilist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "Priority"); - - feeUnlikely = CFeeRate(0); - feeLikely = CFeeRate(INF_FEERATE); - priUnlikely = 0; - priLikely = INF_PRIORITY; -} - -bool CBlockPolicyEstimator::isFeeDataPoint(const CFeeRate &fee, double pri) -{ - if ((pri < minTrackedPriority && fee >= minTrackedFee) || - (pri < priUnlikely && fee > feeLikely)) { - return true; - } - return false; -} - -bool CBlockPolicyEstimator::isPriDataPoint(const CFeeRate &fee, double pri) -{ - if ((fee < minTrackedFee && pri >= minTrackedPriority) || - (fee < feeUnlikely && pri > priLikely)) { - return true; - } - return false; + feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate) { unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); - if (mapMemPoolTxs[hash].stats != NULL) { + if (mapMemPoolTxs.count(hash)) { LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); return; @@ -371,30 +336,11 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo return; } - // Fees are stored and reported as BTC-per-kb: + // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); - // Want the priority of the tx at confirmation. However we don't know - // what that will be and its too hard to continue updating it - // so use starting priority as a proxy - double curPri = entry.GetPriority(txHeight); mapMemPoolTxs[hash].blockHeight = txHeight; - - LogPrint("estimatefee", "Blockpolicy mempool tx %s ", hash.ToString().substr(0,10)); - // Record this as a priority estimate - if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { - mapMemPoolTxs[hash].stats = &priStats; - mapMemPoolTxs[hash].bucketIndex = priStats.NewTx(txHeight, curPri); - } - // Record this as a fee estimate - else if (isFeeDataPoint(feeRate, curPri)) { - mapMemPoolTxs[hash].stats = &feeStats; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); - } - else { - LogPrint("estimatefee", "not adding"); - } - LogPrint("estimatefee", "\n"); + mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); } void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry) @@ -417,21 +363,10 @@ void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM return; } - // Fees are stored and reported as BTC-per-kb: + // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); - // Want the priority of the tx at confirmation. The priority when it - // entered the mempool could easily be very small and change quickly - double curPri = entry.GetPriority(nBlockHeight); - - // Record this as a priority estimate - if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { - priStats.Record(blocksToConfirm, curPri); - } - // Record this as a fee estimate - else if (isFeeDataPoint(feeRate, curPri)) { - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); - } + feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, @@ -452,41 +387,15 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, if (!fCurrentEstimate) return; - // Update the dynamic cutoffs - // a fee/priority is "likely" the reason your tx was included in a block if >85% of such tx's - // were confirmed in 2 blocks and is "unlikely" if <50% were confirmed in 10 blocks - LogPrint("estimatefee", "Blockpolicy recalculating dynamic cutoffs:\n"); - priLikely = priStats.EstimateMedianVal(2, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBlockHeight); - if (priLikely == -1) - priLikely = INF_PRIORITY; - - double feeLikelyEst = feeStats.EstimateMedianVal(2, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBlockHeight); - if (feeLikelyEst == -1) - feeLikely = CFeeRate(INF_FEERATE); - else - feeLikely = CFeeRate(feeLikelyEst); - - priUnlikely = priStats.EstimateMedianVal(10, SUFFICIENT_PRITXS, UNLIKELY_PCT, false, nBlockHeight); - if (priUnlikely == -1) - priUnlikely = 0; - - double feeUnlikelyEst = feeStats.EstimateMedianVal(10, SUFFICIENT_FEETXS, UNLIKELY_PCT, false, nBlockHeight); - if (feeUnlikelyEst == -1) - feeUnlikely = CFeeRate(0); - else - feeUnlikely = CFeeRate(feeUnlikelyEst); - - // Clear the current block states + // Clear the current block state feeStats.ClearCurrent(nBlockHeight); - priStats.ClearCurrent(nBlockHeight); // Repopulate the current block states for (unsigned int i = 0; i < entries.size(); i++) processBlockTx(nBlockHeight, entries[i]); - // Update all exponential averages with the current block states + // Update all exponential averages with the current block state feeStats.UpdateMovingAverages(); - priStats.UpdateMovingAverages(); LogPrint("estimatefee", "Blockpolicy after updating estimates for %u confirmed entries, new mempool map size %u\n", entries.size(), mapMemPoolTxs.size()); @@ -522,7 +431,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun if (answerFoundAtTarget) *answerFoundAtTarget = confTarget - 1; - // If mempool is limiting txs , return at least the min fee from the mempool + // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); if (minPoolFee > 0 && minPoolFee > median) return CFeeRate(minPoolFee); @@ -535,51 +444,38 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun double CBlockPolicyEstimator::estimatePriority(int confTarget) { - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) - return -1; - - return priStats.EstimateMedianVal(confTarget, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + return -1; } double CBlockPolicyEstimator::estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) - return -1; // If mempool is limiting txs, no priority txs are allowed CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); if (minPoolFee > 0) return INF_PRIORITY; - double median = -1; - while (median < 0 && (unsigned int)confTarget <= priStats.GetMaxConfirms()) { - median = priStats.EstimateMedianVal(confTarget++, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } - - if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; - - return median; + return -1; } void CBlockPolicyEstimator::Write(CAutoFile& fileout) { fileout << nBestSeenHeight; feeStats.Write(fileout); - priStats.Write(fileout); } -void CBlockPolicyEstimator::Read(CAutoFile& filein) +void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) { int nFileBestSeenHeight; filein >> nFileBestSeenHeight; feeStats.Read(filein); - priStats.Read(filein); nBestSeenHeight = nFileBestSeenHeight; + if (nFileVersion < 139900) { + TxConfirmStats priStats; + priStats.Read(filein); + } } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index 2c1ac3b934..ea4c70e616 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICYESTIMATOR_H @@ -19,60 +19,50 @@ class CTxMemPoolEntry; class CTxMemPool; /** \class CBlockPolicyEstimator - * The BlockPolicyEstimator is used for estimating the fee or priority needed + * The BlockPolicyEstimator is used for estimating the feerate needed * for a transaction to be included in a block within a certain number of * blocks. * * At a high level the algorithm works by grouping transactions into buckets - * based on having similar priorities or fees and then tracking how long it + * based on having similar feerates and then tracking how long it * takes transactions in the various buckets to be mined. It operates under - * the assumption that in general transactions of higher fee/priority will be - * included in blocks before transactions of lower fee/priority. So for - * example if you wanted to know what fee you should put on a transaction to + * the assumption that in general transactions of higher feerate will be + * included in blocks before transactions of lower feerate. So for + * example if you wanted to know what feerate you should put on a transaction to * be included in a block within the next 5 blocks, you would start by looking - * at the bucket with the highest fee transactions and verifying that a + * at the bucket with the highest feerate transactions and verifying that a * sufficiently high percentage of them were confirmed within 5 blocks and - * then you would look at the next highest fee bucket, and so on, stopping at - * the last bucket to pass the test. The average fee of transactions in this - * bucket will give you an indication of the lowest fee you can put on a + * then you would look at the next highest feerate bucket, and so on, stopping at + * the last bucket to pass the test. The average feerate of transactions in this + * bucket will give you an indication of the lowest feerate you can put on a * transaction and still have a sufficiently high chance of being confirmed * within your desired 5 blocks. * - * When a transaction enters the mempool or is included within a block we - * decide whether it can be used as a data point for fee estimation, priority - * estimation or neither. If the value of exactly one of those properties was - * below the required minimum it can be used to estimate the other. In - * addition, if a priori our estimation code would indicate that the - * transaction would be much more quickly included in a block because of one - * of the properties compared to the other, we can also decide to use it as - * an estimate for that property. - * - * Here is a brief description of the implementation for fee estimation. - * When a transaction that counts for fee estimation enters the mempool, we + * Here is a brief description of the implementation: + * When a transaction enters the mempool, we * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of fee + * we count the number of transactions in each bucket and the total amount of feerate * paid in each bucket. Then we calculate how many blocks Y it took each * transaction to be mined and we track an array of counters in each bucket * for how long it to took transactions to get confirmed from 1 to a max of 25 * and we increment all the counters from Y up to 25. This is because for any * number Z>=Y the transaction was successfully mined within Z blocks. We * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given fee + * counter of the total number of transactions that happened in a given feerate * bucket and the total number that were confirmed in each number 1-25 blocks * or less for any bucket. We save this history by keeping an exponentially * decaying moving average of each one of these stats. Furthermore we also * keep track of the number unmined (in mempool) transactions in each bucket * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that fee bucket when calculating + * the number of transactions we've seen in that feerate bucket when calculating * an estimate for any number of confirmations below the number of blocks * they've been outstanding. */ /** - * We will instantiate two instances of this class, one to track transactions - * that were included in a block due to fee, and one for tx's included due to - * priority. We will lump transactions into a bucket according to their approximate - * fee or priority and then track how long it took for those txs to be included in a block + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block * * The tracking of unconfirmed (mempool) transactions is completely independent of the * historical tracking of transactions that have been confirmed in a block. @@ -80,7 +70,7 @@ class CTxMemPool; class TxConfirmStats { private: - //Define the buckets we will group transactions into (both fee buckets and priority buckets) + //Define the buckets we will group transactions into std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket @@ -97,16 +87,15 @@ private: // and calculate the totals for the current block to update the moving averages std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - // Sum the total priority/fee of all tx's in each bucket + // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector<double> avg; // and calculate the total for the current block to update the moving average std::vector<double> curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg fee/priority per bucket + // Combine the total value with the tx counts to calculate the avg feerate per bucket - std::string dataTypeString; double decay; // Mempool counts of outstanding transactions @@ -123,9 +112,8 @@ public: * @param defaultBuckets contains the upper limits for the bucket boundaries * @param maxConfirms max number of confirms to track * @param decay how much to decay the historical moving average per block - * @param dataTypeString for logging purposes */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay, std::string dataTypeString); + void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); /** Clear the state of the curBlock variables to start counting for the new block */ void ClearCurrent(unsigned int nBlockHeight); @@ -133,7 +121,7 @@ public: /** * Record a new transaction data point in the current block stats * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val either the fee or the priority when entered of the transaction + * @param val the feerate of the transaction * @warning blocksToConfirm is 1-based and has to be >= 1 */ void Record(int blocksToConfirm, double val); @@ -150,14 +138,14 @@ public: void UpdateMovingAverages(); /** - * Calculate a fee or priority estimate. Find the lowest value bucket (or range of buckets + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets * to make sure we have enough data points) whose transactions still have sufficient likelihood * of being confirmed within the target number of confirmations * @param confTarget target number of confirmations * @param sufficientTxVal required average number of transactions per block in a bucket range * @param minSuccess the success probability we require - * @param requireGreater return the lowest fee/pri such that all higher values pass minSuccess OR - * return the highest fee/pri such that all lower values fail minSuccess + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, @@ -184,35 +172,26 @@ static const unsigned int MAX_BLOCK_CONFIRMS = 25; /** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ static const double DEFAULT_DECAY = .998; -/** Require greater than 95% of X fee transactions to be confirmed within Y blocks for X to be big enough */ +/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ static const double MIN_SUCCESS_PCT = .95; -static const double UNLIKELY_PCT = .5; -/** Require an avg of 1 tx in the combined fee bucket per block to have stat significance */ +/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ static const double SUFFICIENT_FEETXS = 1; -/** Require only an avg of 1 tx every 5 blocks in the combined pri bucket (way less pri txs) */ -static const double SUFFICIENT_PRITXS = .2; - -// Minimum and Maximum values for tracking fees and priorities +// Minimum and Maximum values for tracking feerates static const double MIN_FEERATE = 10; static const double MAX_FEERATE = 1e7; static const double INF_FEERATE = MAX_MONEY; -static const double MIN_PRIORITY = 10; -static const double MAX_PRIORITY = 1e16; static const double INF_PRIORITY = 1e9 * MAX_MONEY; -// We have to lump transactions into buckets based on fee or priority, but we want to be able -// to give accurate estimates over a large range of potential fees and priorities +// We have to lump transactions into buckets based on feerate, but we want to be able +// to give accurate estimates over a large range of potential feerates // Therefore it makes sense to exponentially space the buckets /** Spacing of FeeRate buckets */ static const double FEE_SPACING = 1.1; -/** Spacing of Priority buckets */ -static const double PRI_SPACING = 2; - /** - * We want to be able to estimate fees or priorities that are needed on tx's to be included in + * We want to be able to estimate feerates that are needed on tx's to be included in * a certain number of blocks. Every time a block is added to the best chain, this class records * stats on the transactions included in that block */ @@ -235,27 +214,26 @@ public: /** Remove a transaction from the mempool tracking stats*/ void removeTx(uint256 hash); - /** Is this transaction likely included in a block because of its fee?*/ - bool isFeeDataPoint(const CFeeRate &fee, double pri); - - /** Is this transaction likely included in a block because of its priority?*/ - bool isPriDataPoint(const CFeeRate &fee, double pri); - - /** Return a fee estimate */ + /** Return a feerate estimate */ CFeeRate estimateFee(int confTarget); - /** Estimate fee rate needed to get be included in a block within + /** Estimate feerate needed to get be included in a block within * confTarget blocks. If no answer can be given at confTarget, return an * estimate at the lowest target where one can be given. */ CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); - /** Return a priority estimate */ + /** Return a priority estimate. + * DEPRECATED + * Returns -1 + */ double estimatePriority(int confTarget); /** Estimate priority needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + * confTarget blocks. + * DEPRECATED + * Returns -1 unless mempool is currently limited then returns INF_PRIORITY + * answerFoundAtTarget is set to confTarget */ double estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); @@ -263,29 +241,23 @@ public: void Write(CAutoFile& fileout); /** Read estimation data from a file */ - void Read(CAutoFile& filein); + void Read(CAutoFile& filein, int nFileVersion); private: CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main - double minTrackedPriority; //!< Set to AllowFreeThreshold unsigned int nBestSeenHeight; struct TxStatsInfo { - TxConfirmStats *stats; unsigned int blockHeight; unsigned int bucketIndex; - TxStatsInfo() : stats(NULL), blockHeight(0), bucketIndex(0) {} + TxStatsInfo() : blockHeight(0), bucketIndex(0) {} }; // map of txids to information about that transaction std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats, priStats; - - /** Breakpoints to help determine whether a transaction was confirmed by priority or Fee */ - CFeeRate feeLikely, feeUnlikely; - double priLikely, priUnlikely; + TxConfirmStats feeStats; }; class FeeFilterRounder diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ae42b2bd74..3ad1ff7bae 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin developers +// Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -66,7 +66,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes // Extremely large transactions with lots of inputs can cost the network // almost as much to process as they cost the sender in fees, because // computing signature hashes is O(ninputs*txsize). Limiting transactions - // to MAX_STANDARD_TX_SIZE mitigates CPU exhaustion attacks. + // to MAX_STANDARD_TX_WEIGHT mitigates CPU exhaustion attacks. unsigned int sz = GetTransactionWeight(tx); if (sz >= MAX_STANDARD_TX_WEIGHT) { reason = "tx-size"; diff --git a/src/policy/policy.h b/src/policy/policy.h index 814e6c0b6f..764ee27806 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin developers +// Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 133cff611d..d9b47e71bb 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 5a711dba07..139aec5760 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 0e6ab4dd71..95bd2211f9 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -27,7 +27,7 @@ std::string CBlock::ToString() const vtx.size()); for (unsigned int i = 0; i < vtx.size(); i++) { - s << " " << vtx[i].ToString() << "\n"; + s << " " << vtx[i]->ToString() << "\n"; } return s.str(); } diff --git a/src/primitives/block.h b/src/primitives/block.h index 72dfed985a..b037fc839c 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -36,7 +36,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(this->nVersion); READWRITE(hashPrevBlock); READWRITE(hashMerkleRoot); @@ -73,7 +73,7 @@ class CBlock : public CBlockHeader { public: // network and disk - std::vector<CTransaction> vtx; + std::vector<CTransactionRef> vtx; // memory only mutable bool fChecked; @@ -92,7 +92,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(*(CBlockHeader*)this); READWRITE(vtx); } @@ -137,8 +137,9 @@ struct CBlockLocator ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); READWRITE(vHave); } diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 4afbe99fd3..91f4d29488 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -49,11 +49,6 @@ CTxOut::CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn) scriptPubKey = scriptPubKeyIn; } -uint256 CTxOut::GetHash() const -{ - return SerializeHash(*this); -} - std::string CTxOut::ToString() const { return strprintf("CTxOut(nValue=%d.%08d, scriptPubKey=%s)", nValue / COIN, nValue % COIN, HexStr(scriptPubKey).substr(0, 30)); @@ -83,6 +78,10 @@ CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion UpdateHash(); } +CTransaction::CTransaction(CMutableTransaction &&tx) : nVersion(tx.nVersion), vin(std::move(tx.vin)), vout(std::move(tx.vout)), wit(std::move(tx.wit)), nLockTime(tx.nLockTime) { + UpdateHash(); +} + CTransaction& CTransaction::operator=(const CTransaction &tx) { *const_cast<int*>(&nVersion) = tx.nVersion; *const_cast<std::vector<CTxIn>*>(&vin) = tx.vin; diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 16c2e5c454..0fa85a1519 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -28,7 +28,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(hash); READWRITE(n); } @@ -104,7 +104,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(prevout); READWRITE(*(CScriptBase*)(&scriptSig)); READWRITE(nSequence); @@ -144,7 +144,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nValue); READWRITE(*(CScriptBase*)(&scriptPubKey)); } @@ -160,8 +160,6 @@ public: return (nValue == -1); } - uint256 GetHash() const; - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const { // "Dust" is defined in terms of CTransaction::minRelayTxFee, @@ -179,7 +177,7 @@ public: if (scriptPubKey.IsUnspendable()) return 0; - size_t nSize = GetSerializeSize(SER_DISK, 0); + size_t nSize = GetSerializeSize(*this, SER_DISK, 0); int witnessversion = 0; std::vector<unsigned char> witnessprogram; @@ -221,7 +219,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(scriptWitness.stack); } @@ -257,7 +255,7 @@ public: } template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { for (size_t n = 0; n < vtxinwit.size(); n++) { READWRITE(vtxinwit[n]); @@ -289,8 +287,8 @@ struct CMutableTransaction; * - uint32_t nLockTime */ template<typename Stream, typename Operation, typename TxType> -inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action, int nType, int nVersion) { - const bool fAllowWitness = !(nVersion & SERIALIZE_TRANSACTION_NO_WITNESS); +inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action) { + const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); READWRITE(*const_cast<int32_t*>(&tx.nVersion)); unsigned char flags = 0; @@ -381,19 +379,23 @@ public: /** Convert a CMutableTransaction into a CTransaction. */ CTransaction(const CMutableTransaction &tx); + CTransaction(CMutableTransaction &&tx); CTransaction& operator=(const CTransaction& tx); ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - SerializeTransaction(*this, s, ser_action, nType, nVersion); + inline void SerializationOp(Stream& s, Operation ser_action) { + SerializeTransaction(*this, s, ser_action); if (ser_action.ForRead()) { UpdateHash(); } } + template <typename Stream> + CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} + bool IsNull() const { return vin.empty() && vout.empty(); } @@ -458,8 +460,13 @@ struct CMutableTransaction ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - SerializeTransaction(*this, s, ser_action, nType, nVersion); + inline void SerializationOp(Stream& s, Operation ser_action) { + SerializeTransaction(*this, s, ser_action); + } + + template <typename Stream> + CMutableTransaction(deserialize_type, Stream& s) { + Unserialize(s); } /** Compute the hash of this CMutableTransaction. This is computed on the @@ -468,6 +475,12 @@ struct CMutableTransaction uint256 GetHash() const; }; +typedef std::shared_ptr<const CTransaction> CTransactionRef; +static inline CTransactionRef MakeTransactionRef() { return std::make_shared<const CTransaction>(); } +template <typename Tx> static inline CTransactionRef MakeTransactionRef(Tx&& txIn) { return std::make_shared<const CTransaction>(std::forward<Tx>(txIn)); } +static inline CTransactionRef MakeTransactionRef(const CTransactionRef& txIn) { return txIn; } +static inline CTransactionRef MakeTransactionRef(CTransactionRef&& txIn) { return std::move(txIn); } + /** Compute the weight of a transaction, as defined by BIP 141 */ int64_t GetTransactionWeight(const CTransaction &tx); diff --git a/src/protocol.cpp b/src/protocol.cpp index 54ad62b1a2..87d6e06848 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -181,7 +181,11 @@ std::string CInv::GetCommand() const std::string CInv::ToString() const { - return strprintf("%s %s", GetCommand(), hash.ToString()); + try { + return strprintf("%s %s", GetCommand(), hash.ToString()); + } catch(const std::out_of_range &) { + return strprintf("0x%08x %s", type, hash.ToString()); + } } const std::vector<std::string> &getAllNetMessageTypes() diff --git a/src/protocol.h b/src/protocol.h index d19e0d3a5e..dea409c5b7 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -48,7 +48,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(FLATDATA(pchMessageStart)); READWRITE(FLATDATA(pchCommand)); @@ -261,7 +261,7 @@ enum ServiceFlags : uint64_t { // Bitcoin Core nodes used to support this by default, without advertising this bit, // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) NODE_BLOOM = (1 << 2), - // Indicates that a node can be asked for blocks and transactions including + // NODE_WITNESS indicates that a node can be asked for blocks and transactions including // witness data. NODE_WITNESS = (1 << 3), // NODE_XTHIN means the node supports Xtreme Thinblocks @@ -289,14 +289,15 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { if (ser_action.ForRead()) Init(); - if (nType & SER_DISK) + int nVersion = s.GetVersion(); + if (s.GetType() & SER_DISK) READWRITE(nVersion); - if ((nType & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(nType & SER_GETHASH))) + if ((s.GetType() & SER_DISK) || + (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) READWRITE(nTime); uint64_t nServicesInt = nServices; READWRITE(nServicesInt); @@ -343,7 +344,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(type); READWRITE(hash); diff --git a/src/pubkey.h b/src/pubkey.h index 3a554877f8..9499862210 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -116,19 +116,15 @@ public: } //! Implement serialization, as if this was a byte vector. - unsigned int GetSerializeSize(int nType, int nVersion) const - { - return size() + 1; - } template <typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { unsigned int len = size(); ::WriteCompactSize(s, len); s.write((char*)vch, len); } template <typename Stream> - void Unserialize(Stream& s, int nType, int nVersion) + void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); if (len <= 65) { @@ -214,12 +210,13 @@ struct CExtPubKey { void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; - unsigned int GetSerializeSize(int nType, int nVersion) const + void Serialize(CSizeComputer& s) const { - return BIP32_EXTKEY_SIZE+1; //add one byte for the size (compact int) + // Optimized implementation for ::GetSerializeSize that avoids copying. + s.seek(BIP32_EXTKEY_SIZE + 1); // add one byte for the size (compact int) } template <typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { unsigned int len = BIP32_EXTKEY_SIZE; ::WriteCompactSize(s, len); @@ -228,7 +225,7 @@ struct CExtPubKey { s.write((const char *)&code[0], len); } template <typename Stream> - void Unserialize(Stream& s, int nType, int nVersion) + void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); unsigned char code[BIP32_EXTKEY_SIZE]; diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 58cf4dede0..6076837fde 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -83,7 +83,7 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, deleteAction = new QAction(ui->deleteAddress->text(), this); // Build context menu - contextMenu = new QMenu(); + contextMenu = new QMenu(this); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(editAction); diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 6e11e23904..fe53d89ba9 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -87,7 +87,7 @@ BanTableModel::BanTableModel(ClientModel *parent) : clientModel(parent) { columns << tr("IP/Netmask") << tr("Banned Until"); - priv = new BanTablePriv(); + priv.reset(new BanTablePriv()); // default to unsorted priv->sortColumn = -1; @@ -95,6 +95,11 @@ BanTableModel::BanTableModel(ClientModel *parent) : refresh(); } +BanTableModel::~BanTableModel() +{ + // Intentionally left empty +} + int BanTableModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent); diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index fe9600ac0b..3c03d05c05 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -40,6 +40,7 @@ class BanTableModel : public QAbstractTableModel public: explicit BanTableModel(ClientModel *parent = 0); + ~BanTableModel(); void startAutoRefresh(); void stopAutoRefresh(); @@ -66,7 +67,7 @@ public Q_SLOTS: private: ClientModel *clientModel; QStringList columns; - BanTablePriv *priv; + std::unique_ptr<BanTablePriv> priv; }; #endif // BITCOIN_QT_BANTABLEMODEL_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 5e1a41d5a7..4f48e21a22 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -245,6 +245,7 @@ private: #endif int returnValue; const PlatformStyle *platformStyle; + std::unique_ptr<QWidget> shutdownWindow; void startThread(); }; @@ -380,9 +381,8 @@ void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) { SplashScreen *splash = new SplashScreen(0, networkStyle); - // We don't hold a direct pointer to the splash screen after creation, so use - // Qt::WA_DeleteOnClose to make sure that the window will be deleted eventually. - splash->setAttribute(Qt::WA_DeleteOnClose); + // We don't hold a direct pointer to the splash screen after creation, but the splash + // screen will take care of deleting itself when slotFinish happens. splash->show(); connect(this, SIGNAL(splashFinished(QWidget*)), splash, SLOT(slotFinish(QWidget*))); connect(this, SIGNAL(requestedShutdown()), splash, SLOT(close())); @@ -424,6 +424,11 @@ void BitcoinApplication::requestInitialize() void BitcoinApplication::requestShutdown() { + // Show a simple window indicating shutdown status + // Do this first as some of the steps may take some time below, + // for example the RPC console may still be executing a command. + shutdownWindow.reset(ShutdownWindow::showShutdownWindow(window)); + qDebug() << __func__ << ": Requesting shutdown"; startThread(); window->hide(); @@ -438,9 +443,6 @@ void BitcoinApplication::requestShutdown() delete clientModel; clientModel = 0; - // Show a simple window indicating shutdown status - ShutdownWindow::showShutdownWindow(window); - // Request shutdown from core thread Q_EMIT requestedShutdown(); } @@ -511,7 +513,7 @@ void BitcoinApplication::shutdownResult(int retval) void BitcoinApplication::handleRunawayException(const QString &message) { QMessageBox::critical(0, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. Bitcoin can no longer continue safely and will quit.") + QString("\n\n") + message); - ::exit(1); + ::exit(EXIT_FAILURE); } WId BitcoinApplication::getMainWinId() const @@ -588,13 +590,13 @@ int main(int argc, char *argv[]) { HelpMessageDialog help(NULL, mapArgs.count("-version")); help.showOrPrint(); - return 1; + return EXIT_SUCCESS; } /// 5. Now that settings and translations are available, ask user for data directory // User language is set up: pick a data directory if (!Intro::pickDataDirectory()) - return 0; + return EXIT_SUCCESS; /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes @@ -602,14 +604,14 @@ int main(int argc, char *argv[]) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(mapArgs["-datadir"]))); - return 1; + return EXIT_FAILURE; } try { ReadConfigFile(GetArg("-conf", BITCOIN_CONF_FILENAME), mapArgs, mapMultiArgs); } catch (const std::exception& e) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: Cannot parse configuration file: %1. Only use key=value syntax.").arg(e.what())); - return false; + return EXIT_FAILURE; } /// 7. Determine network (and switch to network specific options) @@ -623,7 +625,7 @@ int main(int argc, char *argv[]) SelectParams(ChainNameFromCommandLine()); } catch(std::exception &e) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: %1").arg(e.what())); - return 1; + return EXIT_FAILURE; } #ifdef ENABLE_WALLET // Parse URIs on command line -- this can affect Params() @@ -645,7 +647,7 @@ int main(int argc, char *argv[]) // - Do this after creating app and setting up translations, so errors are // translated properly. if (PaymentServer::ipcSendCommandLine()) - exit(0); + exit(EXIT_SUCCESS); // Start up the payment server early, too, so impatient users that click on // bitcoin: links repeatedly have their payment requests routed to this process: diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index ca5b1fa673..451d391237 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -52,6 +52,7 @@ <file alias="transaction_abandoned">res/icons/transaction_abandoned.png</file> <file alias="hd_enabled">res/icons/hd_enabled.png</file> <file alias="hd_disabled">res/icons/hd_disabled.png</file> + <file alias="network_disabled">res/icons/network_disabled.png</file> </qresource> <qresource prefix="/movies"> <file alias="spinner-000">res/movies/spinner-000.png</file> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index ee5102c4f9..6112a1d256 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -86,7 +86,7 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * unitDisplayControl(0), labelWalletEncryptionIcon(0), labelWalletHDStatusIcon(0), - labelConnectionsIcon(0), + connectionsControl(0), labelBlocksIcon(0), progressBarLabel(0), progressBar(0), @@ -199,7 +199,7 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * unitDisplayControl = new UnitDisplayStatusBarControl(platformStyle); labelWalletEncryptionIcon = new QLabel(); labelWalletHDStatusIcon = new QLabel(); - labelConnectionsIcon = new QLabel(); + connectionsControl = new NetworkToggleStatusBarControl(); labelBlocksIcon = new QLabel(); if(enableWallet) { @@ -210,7 +210,7 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *_platformStyle, const NetworkStyle * frameBlocksLayout->addWidget(labelWalletHDStatusIcon); } frameBlocksLayout->addStretch(); - frameBlocksLayout->addWidget(labelConnectionsIcon); + frameBlocksLayout->addWidget(connectionsControl); frameBlocksLayout->addStretch(); frameBlocksLayout->addWidget(labelBlocksIcon); frameBlocksLayout->addStretch(); @@ -469,8 +469,9 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) createTrayIconMenu(); // Keep up to date with client - setNumConnections(_clientModel->getNumConnections()); + updateNetworkState(); connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); + connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -489,6 +490,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) } #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(_clientModel->getOptionsModel()); + connectionsControl->setClientModel(_clientModel); OptionsModel* optionsModel = _clientModel->getOptionsModel(); if(optionsModel) @@ -509,6 +511,13 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // Disable context menu on tray icon trayIconMenu->clear(); } + // Propagate cleared model to child objects + rpcConsole->setClientModel(nullptr); +#ifdef ENABLE_WALLET + walletFrame->setClientModel(nullptr); +#endif // ENABLE_WALLET + unitDisplayControl->setOptionsModel(nullptr); + connectionsControl->setClientModel(nullptr); } } @@ -698,8 +707,9 @@ void BitcoinGUI::gotoVerifyMessageTab(QString addr) } #endif // ENABLE_WALLET -void BitcoinGUI::setNumConnections(int count) +void BitcoinGUI::updateNetworkState() { + int count = clientModel->getNumConnections(); QString icon; switch(count) { @@ -709,8 +719,31 @@ void BitcoinGUI::setNumConnections(int count) case 7: case 8: case 9: icon = ":/icons/connect_3"; break; default: icon = ":/icons/connect_4"; break; } - labelConnectionsIcon->setPixmap(platformStyle->SingleColorIcon(icon).pixmap(STATUSBAR_ICONSIZE,STATUSBAR_ICONSIZE)); - labelConnectionsIcon->setToolTip(tr("%n active connection(s) to Bitcoin network", "", count)); + + QString tooltip; + + if (clientModel->getNetworkActive()) { + tooltip = tr("%n active connection(s) to Bitcoin network", "", count) + QString(".<br>") + tr("Click to disable network activity."); + } else { + tooltip = tr("Network activity disabled.") + QString("<br>") + tr("Click to enable network activity again."); + icon = ":/icons/network_disabled"; + } + + // Don't word-wrap this (fixed-width) tooltip + tooltip = QString("<nobr>") + tooltip + QString("</nobr>"); + connectionsControl->setToolTip(tooltip); + + connectionsControl->setPixmap(platformStyle->SingleColorIcon(icon).pixmap(STATUSBAR_ICONSIZE,STATUSBAR_ICONSIZE)); +} + +void BitcoinGUI::setNumConnections(int count) +{ + updateNetworkState(); +} + +void BitcoinGUI::setNetworkActive(bool networkActive) +{ + updateNetworkState(); } void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header) @@ -1165,7 +1198,7 @@ void UnitDisplayStatusBarControl::mousePressEvent(QMouseEvent *event) /** Creates context menu, its actions, and wires up all the relevant signals for mouse events. */ void UnitDisplayStatusBarControl::createContextMenu() { - menu = new QMenu(); + menu = new QMenu(this); Q_FOREACH(BitcoinUnits::Unit u, BitcoinUnits::availableUnits()) { QAction *menuAction = new QAction(QString(BitcoinUnits::name(u)), this); @@ -1211,3 +1244,16 @@ void UnitDisplayStatusBarControl::onMenuSelection(QAction* action) optionsModel->setDisplayUnit(action->data()); } } + +void NetworkToggleStatusBarControl::mousePressEvent(QMouseEvent *event) +{ + if (clientModel) { + clientModel->setNetworkActive(!clientModel->getNetworkActive()); + } +} + +/** Lets the control know about the Client Model */ +void NetworkToggleStatusBarControl::setClientModel(ClientModel *_clientModel) +{ + this->clientModel = _clientModel; +} diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 0eaa44b263..1b02e77fc4 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -26,6 +26,7 @@ class PlatformStyle; class RPCConsole; class SendCoinsRecipient; class UnitDisplayStatusBarControl; +class NetworkToggleStatusBarControl; class WalletFrame; class WalletModel; class HelpMessageDialog; @@ -85,7 +86,7 @@ private: UnitDisplayStatusBarControl *unitDisplayControl; QLabel *labelWalletEncryptionIcon; QLabel *labelWalletHDStatusIcon; - QLabel *labelConnectionsIcon; + NetworkToggleStatusBarControl *connectionsControl; QLabel *labelBlocksIcon; QLabel *progressBarLabel; QProgressBar *progressBar; @@ -146,6 +147,9 @@ private: /** Disconnect core signals from GUI client */ void unsubscribeFromCoreSignals(); + /** Update UI with latest network info from model. */ + void updateNetworkState(); + Q_SIGNALS: /** Signal raised when a URI was entered or dragged to the GUI */ void receivedURI(const QString &uri); @@ -153,6 +157,8 @@ Q_SIGNALS: public Q_SLOTS: /** Set number of connections shown in the UI */ void setNumConnections(int count); + /** Set network state shown in the UI */ + void setNetworkActive(bool networkActive); /** Set number of blocks and last block date shown in the UI */ void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); @@ -264,4 +270,17 @@ private Q_SLOTS: void onMenuSelection(QAction* action); }; +class NetworkToggleStatusBarControl : public QLabel +{ + Q_OBJECT + +public: + void setClientModel(ClientModel *clientModel); +protected: + void mousePressEvent(QMouseEvent *event); + +private: + ClientModel *clientModel; +}; + #endif // BITCOIN_QT_BITCOINGUI_H diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 87704c641d..a4bb2f77fe 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -6,6 +6,7 @@ #include "bantablemodel.h" #include "guiconstants.h" +#include "guiutil.h" #include "peertablemodel.h" #include "chainparams.h" @@ -144,6 +145,11 @@ void ClientModel::updateNumConnections(int numConnections) Q_EMIT numConnectionsChanged(numConnections); } +void ClientModel::updateNetworkActive(bool networkActive) +{ + Q_EMIT networkActiveChanged(networkActive); +} + void ClientModel::updateAlert() { Q_EMIT alertsChanged(getStatusBarWarnings()); @@ -166,6 +172,21 @@ enum BlockSource ClientModel::getBlockSource() const return BLOCK_SOURCE_NONE; } +void ClientModel::setNetworkActive(bool active) +{ + if (g_connman) { + g_connman->SetNetworkActive(active); + } +} + +bool ClientModel::getNetworkActive() const +{ + if (g_connman) { + return g_connman->GetNetworkActive(); + } + return false; +} + QString ClientModel::getStatusBarWarnings() const { return QString::fromStdString(GetWarnings("gui")); @@ -208,7 +229,7 @@ QString ClientModel::formatClientStartupTime() const QString ClientModel::dataDir() const { - return QString::fromStdString(GetDataDir().string()); + return GUIUtil::boostPathToQString(GetDataDir()); } void ClientModel::updateBanlist() @@ -232,6 +253,12 @@ static void NotifyNumConnectionsChanged(ClientModel *clientmodel, int newNumConn Q_ARG(int, newNumConnections)); } +static void NotifyNetworkActiveChanged(ClientModel *clientmodel, bool networkActive) +{ + QMetaObject::invokeMethod(clientmodel, "updateNetworkActive", Qt::QueuedConnection, + Q_ARG(bool, networkActive)); +} + static void NotifyAlertChanged(ClientModel *clientmodel) { qDebug() << "NotifyAlertChanged"; @@ -272,6 +299,7 @@ void ClientModel::subscribeToCoreSignals() // Connect signals to client uiInterface.ShowProgress.connect(boost::bind(ShowProgress, this, _1, _2)); uiInterface.NotifyNumConnectionsChanged.connect(boost::bind(NotifyNumConnectionsChanged, this, _1)); + uiInterface.NotifyNetworkActiveChanged.connect(boost::bind(NotifyNetworkActiveChanged, this, _1)); uiInterface.NotifyAlertChanged.connect(boost::bind(NotifyAlertChanged, this)); uiInterface.BannedListChanged.connect(boost::bind(BannedListChanged, this)); uiInterface.NotifyBlockTip.connect(boost::bind(BlockTipChanged, this, _1, _2, false)); @@ -283,6 +311,7 @@ void ClientModel::unsubscribeFromCoreSignals() // Disconnect signals from client uiInterface.ShowProgress.disconnect(boost::bind(ShowProgress, this, _1, _2)); uiInterface.NotifyNumConnectionsChanged.disconnect(boost::bind(NotifyNumConnectionsChanged, this, _1)); + uiInterface.NotifyNetworkActiveChanged.disconnect(boost::bind(NotifyNetworkActiveChanged, this, _1)); uiInterface.NotifyAlertChanged.disconnect(boost::bind(NotifyAlertChanged, this)); uiInterface.BannedListChanged.disconnect(boost::bind(BannedListChanged, this)); uiInterface.NotifyBlockTip.disconnect(boost::bind(BlockTipChanged, this, _1, _2, false)); diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 3fd8404cbb..a641401425 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -68,6 +68,10 @@ public: bool inInitialBlockDownload() const; //! Return true if core is importing blocks enum BlockSource getBlockSource() const; + //! Return true if network activity in core is enabled + bool getNetworkActive() const; + //! Toggle network activity state in core + void setNetworkActive(bool active); //! Return warnings to be displayed in status bar QString getStatusBarWarnings() const; @@ -91,6 +95,7 @@ Q_SIGNALS: void numConnectionsChanged(int count); void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header); void mempoolSizeChanged(long count, size_t mempoolSizeInBytes); + void networkActiveChanged(bool networkActive); void alertsChanged(const QString &warnings); void bytesChanged(quint64 totalBytesIn, quint64 totalBytesOut); @@ -103,6 +108,7 @@ Q_SIGNALS: public Q_SLOTS: void updateTimer(); void updateNumConnections(int numConnections); + void updateNetworkActive(bool networkActive); void updateAlert(); void updateBanlist(); }; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1a1671f0ee..d77db39b3c 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -35,6 +35,13 @@ QList<CAmount> CoinControlDialog::payAmounts; CCoinControl* CoinControlDialog::coinControl = new CCoinControl(); bool CoinControlDialog::fSubtractFeeFromAmount = false; +bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { + int column = treeWidget()->sortColumn(); + if (column == CoinControlDialog::COLUMN_AMOUNT || column == CoinControlDialog::COLUMN_DATE || column == CoinControlDialog::COLUMN_CONFIRMATIONS) + return data(column, Qt::UserRole).toLongLong() < other.data(column, Qt::UserRole).toLongLong(); + return QTreeWidgetItem::operator<(other); +} + CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::CoinControlDialog), @@ -52,7 +59,7 @@ CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidge unlockAction = new QAction(tr("Unlock unspent"), this); // we need to enable/disable this // context menu - contextMenu = new QMenu(); + contextMenu = new QMenu(this); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -128,11 +135,9 @@ CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidge ui->treeWidget->setColumnWidth(COLUMN_CONFIRMATIONS, 110); ui->treeWidget->setColumnHidden(COLUMN_TXHASH, true); // store transaction hash in this column, but don't show it ui->treeWidget->setColumnHidden(COLUMN_VOUT_INDEX, true); // store vout index in this column, but don't show it - ui->treeWidget->setColumnHidden(COLUMN_AMOUNT_INT64, true); // store amount int64 in this column, but don't show it - ui->treeWidget->setColumnHidden(COLUMN_DATE_INT64, true); // store date int64 in this column, but don't show it // default view is sorted by amount desc - sortView(COLUMN_AMOUNT_INT64, Qt::DescendingOrder); + sortView(COLUMN_AMOUNT, Qt::DescendingOrder); // restore list mode and sortorder as a convenience feature QSettings settings; @@ -164,15 +169,6 @@ void CoinControlDialog::setModel(WalletModel *_model) } } -// helper function str_pad -QString CoinControlDialog::strPad(QString s, int nPadLength, QString sPadding) -{ - while (s.length() < nPadLength) - s = sPadding + s; - - return s; -} - // ok button void CoinControlDialog::buttonBoxClicked(QAbstractButton* button) { @@ -338,7 +334,7 @@ void CoinControlDialog::sortView(int column, Qt::SortOrder order) sortColumn = column; sortOrder = order; ui->treeWidget->sortItems(column, order); - ui->treeWidget->header()->setSortIndicator(getMappedColumn(sortColumn), sortOrder); + ui->treeWidget->header()->setSortIndicator(sortColumn, sortOrder); } // treeview: clicked on header @@ -346,12 +342,10 @@ void CoinControlDialog::headerSectionClicked(int logicalIndex) { if (logicalIndex == COLUMN_CHECKBOX) // click on most left column -> do nothing { - ui->treeWidget->header()->setSortIndicator(getMappedColumn(sortColumn), sortOrder); + ui->treeWidget->header()->setSortIndicator(sortColumn, sortOrder); } else { - logicalIndex = getMappedColumn(logicalIndex, false); - if (sortColumn == logicalIndex) sortOrder = ((sortOrder == Qt::AscendingOrder) ? Qt::DescendingOrder : Qt::AscendingOrder); else @@ -658,7 +652,7 @@ void CoinControlDialog::updateView() model->listCoins(mapCoins); BOOST_FOREACH(const PAIRTYPE(QString, std::vector<COutput>)& coins, mapCoins) { - QTreeWidgetItem *itemWalletAddress = new QTreeWidgetItem(); + CCoinControlWidgetItem *itemWalletAddress = new CCoinControlWidgetItem(); itemWalletAddress->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); QString sWalletAddress = coins.first; QString sWalletLabel = model->getAddressTableModel()->labelForAddress(sWalletAddress); @@ -686,9 +680,9 @@ void CoinControlDialog::updateView() nSum += out.tx->vout[out.i].nValue; nChildren++; - QTreeWidgetItem *itemOutput; - if (treeMode) itemOutput = new QTreeWidgetItem(itemWalletAddress); - else itemOutput = new QTreeWidgetItem(ui->treeWidget); + CCoinControlWidgetItem *itemOutput; + if (treeMode) itemOutput = new CCoinControlWidgetItem(itemWalletAddress); + else itemOutput = new CCoinControlWidgetItem(ui->treeWidget); itemOutput->setFlags(flgCheckbox); itemOutput->setCheckState(COLUMN_CHECKBOX,Qt::Unchecked); @@ -721,14 +715,15 @@ void CoinControlDialog::updateView() // amount itemOutput->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, out.tx->vout[out.i].nValue)); - itemOutput->setText(COLUMN_AMOUNT_INT64, strPad(QString::number(out.tx->vout[out.i].nValue), 15, " ")); // padding so that sorting works correctly + itemOutput->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)out.tx->vout[out.i].nValue)); // padding so that sorting works correctly // date itemOutput->setText(COLUMN_DATE, GUIUtil::dateTimeStr(out.tx->GetTxTime())); - itemOutput->setText(COLUMN_DATE_INT64, strPad(QString::number(out.tx->GetTxTime()), 20, " ")); + itemOutput->setData(COLUMN_DATE, Qt::UserRole, QVariant((qlonglong)out.tx->GetTxTime())); // confirmations - itemOutput->setText(COLUMN_CONFIRMATIONS, strPad(QString::number(out.nDepth), 8, " ")); + itemOutput->setText(COLUMN_CONFIRMATIONS, QString::number(out.nDepth)); + itemOutput->setData(COLUMN_CONFIRMATIONS, Qt::UserRole, QVariant((qlonglong)out.nDepth)); // transaction hash uint256 txhash = out.tx->GetHash(); @@ -756,7 +751,7 @@ void CoinControlDialog::updateView() { itemWalletAddress->setText(COLUMN_CHECKBOX, "(" + QString::number(nChildren) + ")"); itemWalletAddress->setText(COLUMN_AMOUNT, BitcoinUnits::format(nDisplayUnit, nSum)); - itemWalletAddress->setText(COLUMN_AMOUNT_INT64, strPad(QString::number(nSum), 15, " ")); + itemWalletAddress->setData(COLUMN_AMOUNT, Qt::UserRole, QVariant((qlonglong)nSum)); } } diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 7d73421e3a..7c2269b1eb 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -28,6 +28,17 @@ namespace Ui { #define ASYMP_UTF8 "\xE2\x89\x88" +class CCoinControlWidgetItem : public QTreeWidgetItem +{ +public: + CCoinControlWidgetItem(QTreeWidget *parent, int type = Type) : QTreeWidgetItem(parent, type) {} + CCoinControlWidgetItem(int type = Type) : QTreeWidgetItem(type) {} + CCoinControlWidgetItem(QTreeWidgetItem *parent, int type = Type) : QTreeWidgetItem(parent, type) {} + + bool operator<(const QTreeWidgetItem &other) const; +}; + + class CoinControlDialog : public QDialog { Q_OBJECT @@ -59,13 +70,12 @@ private: const PlatformStyle *platformStyle; - QString strPad(QString, int, QString); void sortView(int, Qt::SortOrder); void updateView(); enum { - COLUMN_CHECKBOX, + COLUMN_CHECKBOX = 0, COLUMN_AMOUNT, COLUMN_LABEL, COLUMN_ADDRESS, @@ -73,30 +83,8 @@ private: COLUMN_CONFIRMATIONS, COLUMN_TXHASH, COLUMN_VOUT_INDEX, - COLUMN_AMOUNT_INT64, - COLUMN_DATE_INT64 }; - - // some columns have a hidden column containing the value used for sorting - int getMappedColumn(int column, bool fVisibleColumn = true) - { - if (fVisibleColumn) - { - if (column == COLUMN_AMOUNT_INT64) - return COLUMN_AMOUNT; - else if (column == COLUMN_DATE_INT64) - return COLUMN_DATE; - } - else - { - if (column == COLUMN_AMOUNT) - return COLUMN_AMOUNT_INT64; - else if (column == COLUMN_DATE) - return COLUMN_DATE_INT64; - } - - return column; - } + friend class CCoinControlWidgetItem; private Q_SLOTS: void showMenu(const QPoint &); diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index 27998f90c5..a37672ad53 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -130,7 +130,7 @@ QLabel { color: rgb(40,40,40); }</string> <item> <widget class="QLabel" name="infoText"> <property name="text"> - <string>The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet. This means that recent transactions will not be visible, and the balance will not be up-to-date until this process has completed.</string> + <string>Recent transactions may not yet be visible, and therefore your wallet's balance might be incorrect. This information will be correct once your wallet has finished synchronizing with the bitcoin network, as detailed below.</string> </property> <property name="textFormat"> <enum>Qt::RichText</enum> @@ -149,7 +149,7 @@ QLabel { color: rgb(40,40,40); }</string> </font> </property> <property name="text"> - <string>Spending bitcoins may not be possible during that phase!</string> + <string>Attempting to spend bitcoins that are affected by not-yet-displayed transactions will not be accepted by the network.</string> </property> <property name="textFormat"> <enum>Qt::RichText</enum> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 06e09074d1..33db9f8938 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -411,7 +411,7 @@ </property> </widget> </item> - </layout> + </layout> </item> <item> <layout class="QFormLayout" name="formLayoutCoinControl4"> @@ -1031,7 +1031,7 @@ <item> <widget class="QLabel" name="labelSmartFee3"> <property name="text"> - <string>Confirmation time:</string> + <string>Confirmation time target:</string> </property> <property name="margin"> <number>2</number> @@ -1096,6 +1096,26 @@ </widget> </item> <item> + <spacer name="horizontalSpacer_7"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QLabel" name="confirmationTargetLabel"> + <property name="text"> + <string>(count)</string> + </property> + </widget> + </item> + <item> <spacer name="horizontalSpacer_3"> <property name="orientation"> <enum>Qt::Horizontal</enum> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 42dafa1175..130cfc6e7d 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -291,17 +291,11 @@ void copyEntryData(QAbstractItemView *view, int column, int role) } } -QVariant getEntryData(QAbstractItemView *view, int column, int role) +QList<QModelIndex> getEntryData(QAbstractItemView *view, int column) { if(!view || !view->selectionModel()) - return QVariant(); - QModelIndexList selection = view->selectionModel()->selectedRows(column); - - if(!selection.isEmpty()) { - // Return first item - return (selection.at(0).data(role)); - } - return QVariant(); + return QList<QModelIndex>(); + return view->selectionModel()->selectedRows(column); } QString getSaveFileName(QWidget *parent, const QString &caption, const QString &dir, @@ -591,7 +585,8 @@ void TableViewLastColumnResizingFixer::on_geometriesChanged() * Initializes all internal variables and prepares the * the resize modes of the last 2 columns of the table and */ -TableViewLastColumnResizingFixer::TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth) : +TableViewLastColumnResizingFixer::TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth, QObject *parent) : + QObject(parent), tableView(table), lastColumnMinimumWidth(lastColMinimumWidth), allColumnsMinimumWidth(allColsMinimumWidth) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index e28f68930f..8f1f3fbb2c 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -67,10 +67,9 @@ namespace GUIUtil /** Return a field of the currently selected entry as a QString. Does nothing if nothing is selected. @param[in] column Data column to extract from the model - @param[in] role Data role to extract from the model @see TransactionView::copyLabel, TransactionView::copyAmount, TransactionView::copyAddress */ - QVariant getEntryData(QAbstractItemView *view, int column, int role); + QList<QModelIndex> getEntryData(QAbstractItemView *view, int column); void setClipboard(const QString& str); @@ -150,7 +149,7 @@ namespace GUIUtil Q_OBJECT public: - TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth); + TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth, QObject *parent); void stretchColumnWidth(int column); private: diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index b23b5f2607..d2e1fb23a3 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -81,7 +81,7 @@ private: int nDisplayUnit; QString strThirdPartyTxUrls; bool fCoinControlFeatures; - /* settings that were overriden by command-line */ + /* settings that were overridden by command-line */ QString strOverriddenByCommandLine; // Add option to list of GUI options overridden through command line/config file diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 7ccdb89c0c..ea1bb2fc94 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -25,8 +25,8 @@ class TxViewDelegate : public QAbstractItemDelegate { Q_OBJECT public: - TxViewDelegate(const PlatformStyle *_platformStyle): - QAbstractItemDelegate(), unit(BitcoinUnits::BTC), + TxViewDelegate(const PlatformStyle *_platformStyle, QObject *parent=nullptr): + QAbstractItemDelegate(parent), unit(BitcoinUnits::BTC), platformStyle(_platformStyle) { @@ -119,8 +119,7 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) currentWatchOnlyBalance(-1), currentWatchUnconfBalance(-1), currentWatchImmatureBalance(-1), - txdelegate(new TxViewDelegate(platformStyle)), - filter(0) + txdelegate(new TxViewDelegate(platformStyle, this)) { ui->setupUi(this); @@ -220,7 +219,7 @@ void OverviewPage::setWalletModel(WalletModel *model) if(model && model->getOptionsModel()) { // Set up transaction list - filter = new TransactionFilterProxy(); + filter.reset(new TransactionFilterProxy()); filter->setSourceModel(model->getTransactionTableModel()); filter->setLimit(NUM_ITEMS); filter->setDynamicSortFilter(true); @@ -228,7 +227,7 @@ void OverviewPage::setWalletModel(WalletModel *model) filter->setShowInactive(false); filter->sort(TransactionTableModel::Date, Qt::DescendingOrder); - ui->listTransactions->setModel(filter); + ui->listTransactions->setModel(filter.get()); ui->listTransactions->setModelColumn(TransactionTableModel::ToAddress); // Keep up to date with wallet diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index 65cd3341b6..ffe0de328c 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -8,6 +8,7 @@ #include "amount.h" #include <QWidget> +#include <memory> class ClientModel; class TransactionFilterProxy; @@ -56,7 +57,7 @@ private: CAmount currentWatchImmatureBalance; TxViewDelegate *txdelegate; - TransactionFilterProxy *filter; + std::unique_ptr<TransactionFilterProxy> filter; private Q_SLOTS: void updateDisplayUnit(); diff --git a/src/qt/paymentrequestplus.h b/src/qt/paymentrequestplus.h index a73fe5f29d..a8bfcd2ac4 100644 --- a/src/qt/paymentrequestplus.h +++ b/src/qt/paymentrequestplus.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin developers +// Copyright (c) 2011-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 9f23e77a13..229752cad2 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -58,14 +58,19 @@ const char* BIP71_MIMETYPE_PAYMENTREQUEST = "application/bitcoin-paymentrequest" // BIP70 max payment request size in bytes (DoS protection) const qint64 BIP70_MAX_PAYMENTREQUEST_SIZE = 50000; -X509_STORE* PaymentServer::certStore = NULL; -void PaymentServer::freeCertStore() +struct X509StoreDeleter { + void operator()(X509_STORE* b) { + X509_STORE_free(b); + } +}; + +struct X509Deleter { + void operator()(X509* b) { X509_free(b); } +}; + +namespace // Anon namespace { - if (PaymentServer::certStore != NULL) - { - X509_STORE_free(PaymentServer::certStore); - PaymentServer::certStore = NULL; - } + std::unique_ptr<X509_STORE, X509StoreDeleter> certStore; } // @@ -80,7 +85,7 @@ static QString ipcServerName() // Append a simple hash of the datadir // Note that GetDataDir(true) returns a different path // for -testnet versus main net - QString ddir(QString::fromStdString(GetDataDir(true).string())); + QString ddir(GUIUtil::boostPathToQString(GetDataDir(true))); name.append(QString::number(qHash(ddir))); return name; @@ -107,20 +112,15 @@ static void ReportInvalidCertificate(const QSslCertificate& cert) // void PaymentServer::LoadRootCAs(X509_STORE* _store) { - if (PaymentServer::certStore == NULL) - atexit(PaymentServer::freeCertStore); - else - freeCertStore(); - // Unit tests mostly use this, to pass in fake root CAs: if (_store) { - PaymentServer::certStore = _store; + certStore.reset(_store); return; } // Normal execution, use either -rootcertificates or system certs: - PaymentServer::certStore = X509_STORE_new(); + certStore.reset(X509_STORE_new()); // Note: use "-system-" default here so that users can pass -rootcertificates="" // and get 'I don't like X.509 certificates, don't trust anybody' behavior: @@ -167,11 +167,11 @@ void PaymentServer::LoadRootCAs(X509_STORE* _store) QByteArray certData = cert.toDer(); const unsigned char *data = (const unsigned char *)certData.data(); - X509* x509 = d2i_X509(0, &data, certData.size()); - if (x509 && X509_STORE_add_cert(PaymentServer::certStore, x509)) + std::unique_ptr<X509, X509Deleter> x509(d2i_X509(0, &data, certData.size())); + if (x509 && X509_STORE_add_cert(certStore.get(), x509.get())) { - // Note: X509_STORE_free will free the X509* objects when - // the PaymentServer is destroyed + // Note: X509_STORE increases the reference count to the X509 object, + // we still have to release our reference to it. ++nRootCerts; } else @@ -550,7 +550,7 @@ bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, Sen recipient.paymentRequest = request; recipient.message = GUIUtil::HtmlEscape(request.getDetails().memo()); - request.getMerchant(PaymentServer::certStore, recipient.authenticatedMerchant); + request.getMerchant(certStore.get(), recipient.authenticatedMerchant); QList<std::pair<CScript, CAmount> > sendingTos = request.getPayTo(); QStringList addresses; @@ -807,3 +807,8 @@ bool PaymentServer::verifyAmount(const CAmount& requestAmount) } return fVerified; } + +X509_STORE* PaymentServer::getCertStore() +{ + return certStore.get(); +} diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 2d27ed078b..7202e7dada 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -83,7 +83,7 @@ public: static void LoadRootCAs(X509_STORE* store = NULL); // Return certificate store - static X509_STORE* getCertStore() { return certStore; } + static X509_STORE* getCertStore(); // OptionsModel is used for getting proxy settings and display unit void setOptionsModel(OptionsModel *optionsModel); @@ -140,9 +140,6 @@ private: bool saveURIs; // true during startup QLocalServer* uriServer; - static X509_STORE* certStore; // Trusted root certificates - static void freeCertStore(); - QNetworkAccessManager* netManager; // Used to fetch payment requests OptionsModel *optionsModel; diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index a2f9471fcc..c0b4900285 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -114,12 +114,12 @@ PeerTableModel::PeerTableModel(ClientModel *parent) : timer(0) { columns << tr("NodeId") << tr("Node/Service") << tr("User Agent") << tr("Ping"); - priv = new PeerTablePriv(); + priv.reset(new PeerTablePriv()); // default to unsorted priv->sortColumn = -1; // set up timer for auto refresh - timer = new QTimer(); + timer = new QTimer(this); connect(timer, SIGNAL(timeout()), SLOT(refresh())); timer->setInterval(MODEL_UPDATE_DELAY); @@ -127,6 +127,11 @@ PeerTableModel::PeerTableModel(ClientModel *parent) : refresh(); } +PeerTableModel::~PeerTableModel() +{ + // Intentionally left empty +} + void PeerTableModel::startAutoRefresh() { timer->start(); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index a4f7bbdb3d..af34b147b1 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -46,6 +46,7 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(ClientModel *parent = 0); + ~PeerTableModel(); const CNodeCombinedStats *getNodeStats(int idx); int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); @@ -75,7 +76,7 @@ public Q_SLOTS: private: ClientModel *clientModel; QStringList columns; - PeerTablePriv *priv; + std::unique_ptr<PeerTablePriv> priv; QTimer *timer; }; diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index b50cad4975..dd83547a91 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -25,6 +25,7 @@ ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::ReceiveCoinsDialog), + columnResizingFixer(0), model(0), platformStyle(_platformStyle) { @@ -49,7 +50,7 @@ ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWid QAction *copyAmountAction = new QAction(tr("Copy amount"), this); // context menu - contextMenu = new QMenu(); + contextMenu = new QMenu(this); contextMenu->addAction(copyURIAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyMessageAction); @@ -91,7 +92,7 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) SIGNAL(selectionChanged(QItemSelection, QItemSelection)), this, SLOT(recentRequestsView_selectionChanged(QItemSelection, QItemSelection))); // Last 2 columns are set by the columnResizingFixer, when the table geometry is ready. - columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH); + columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH, this); } } diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 998c9176d7..f896461735 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -32,7 +32,7 @@ QRImageWidget::QRImageWidget(QWidget *parent): QLabel(parent), contextMenu(0) { - contextMenu = new QMenu(); + contextMenu = new QMenu(this); QAction *saveImageAction = new QAction(tr("&Save Image..."), this); connect(saveImageAction, SIGNAL(triggered()), this, SLOT(saveImage())); contextMenu->addAction(saveImageAction); diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 2335d6b282..35d37bb22b 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -14,7 +14,7 @@ #include <boost/foreach.hpp> RecentRequestsTableModel::RecentRequestsTableModel(CWallet *wallet, WalletModel *parent) : - walletModel(parent) + QAbstractTableModel(parent), walletModel(parent) { Q_UNUSED(wallet); nReceiveRequestsMaxId = 0; diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 0193e748d7..8ee2c9cbac 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -27,7 +27,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { unsigned int nDate = date.toTime_t(); READWRITE(this->nVersion); diff --git a/src/qt/res/icons/network_disabled.png b/src/qt/res/icons/network_disabled.png Binary files differnew file mode 100644 index 0000000000..269c3cfab8 --- /dev/null +++ b/src/qt/res/icons/network_disabled.png diff --git a/src/qt/res/src/connect-0.svg b/src/qt/res/src/connect-0.svg index 7d2afac622..0920555b96 100644 --- a/src/qt/res/src/connect-0.svg +++ b/src/qt/res/src/connect-0.svg @@ -7,8 +7,8 @@ xmlns="http://www.w3.org/2000/svg" id="svg2" viewBox="0 0 24 24" - height="24" - width="24" + height="92" + width="92" version="1.2"> <metadata id="metadata10"> diff --git a/src/qt/res/src/connect-1.svg b/src/qt/res/src/connect-1.svg index d17928c97d..25dea4cd3a 100644 --- a/src/qt/res/src/connect-1.svg +++ b/src/qt/res/src/connect-1.svg @@ -6,8 +6,8 @@ xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" version="1.2" - width="24" - height="24" + width="92" + height="92" viewBox="0 0 24 24" id="svg2"> <metadata diff --git a/src/qt/res/src/connect-2.svg b/src/qt/res/src/connect-2.svg index 841ca6071d..bb98333d23 100644 --- a/src/qt/res/src/connect-2.svg +++ b/src/qt/res/src/connect-2.svg @@ -6,8 +6,8 @@ xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" version="1.2" - width="24" - height="24" + width="92" + height="92" viewBox="0 0 24 24" id="svg2"> <metadata diff --git a/src/qt/res/src/connect-3.svg b/src/qt/res/src/connect-3.svg index b06e67daf8..a54a55ef61 100644 --- a/src/qt/res/src/connect-3.svg +++ b/src/qt/res/src/connect-3.svg @@ -7,8 +7,8 @@ xmlns="http://www.w3.org/2000/svg" id="svg2" viewBox="0 0 24 24" - height="24" - width="24" + height="92" + width="92" version="1.2"> <metadata id="metadata10"> diff --git a/src/qt/res/src/connect-4.svg b/src/qt/res/src/connect-4.svg index 0abc7955fd..b83b9f9d03 100644 --- a/src/qt/res/src/connect-4.svg +++ b/src/qt/res/src/connect-4.svg @@ -6,8 +6,8 @@ xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" version="1.2" - width="24" - height="24" + width="92" + height="92" viewBox="0 0 24 24" id="svg2"> <metadata diff --git a/src/qt/res/src/network_disabled.svg b/src/qt/res/src/network_disabled.svg new file mode 100644 index 0000000000..a041d77439 --- /dev/null +++ b/src/qt/res/src/network_disabled.svg @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + id="svg2" + viewBox="0 0 24 24" + width="92" + height="92" + version="1.2"> + <metadata + id="metadata10"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title></dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <defs + id="defs8" /> + <g + id="g4142" + transform="matrix(0,-1,-1,0,23.96,24)"> + <g + id="g4210" + transform="matrix(-1,0,0,1,59.86,-106.6)"> + <g + id="g4289" + transform="matrix(-1,0,0,1,-16.98,0.8136)"> + <g + id="g4291"> + <path + id="path4293" + transform="matrix(0,-1,-1,0,-52.84,129.7464)" + d="M 20.146484 1.0097656 C 18.746484 1.0097656 17.646484 1.8597656 17.146484 3.0097656 L 14.447266 3.0097656 C 12.247266 3.0097656 10.447266 4.7997656 10.447266 7.0097656 L 10.447266 8.1425781 C 10.128283 8.0634395 9.7980674 8.0097656 9.4472656 8.0097656 L 6.8457031 8.0097656 C 6.3457031 6.8597656 5.2457031 6.0097656 3.8457031 6.0097656 C 1.8457031 6.0097656 0.34570312 7.5997656 0.34570312 9.5097656 C 0.34570312 11.419766 1.8457031 13.009766 3.8457031 13.009766 C 5.2457031 13.009766 6.3457031 12.159766 6.8457031 11.009766 L 8.9746094 11.009766 C 8.8693536 11.330059 8.8007812 11.663345 8.8007812 12.001953 C 8.8007813 12.841953 9.1402344 13.671625 9.7402344 14.265625 C 9.9479364 14.475439 10.191281 14.640988 10.447266 14.783203 L 10.447266 16.980469 C 10.447266 17.530469 9.9472656 17.980469 9.4472656 17.980469 L 6.8457031 17.980469 C 6.3457031 16.830469 5.2457031 15.980469 3.8457031 15.980469 C 1.8457031 15.980469 0.34570312 17.570469 0.34570312 19.480469 C 0.34570312 21.390469 1.8457031 22.990234 3.8457031 22.990234 C 5.2457031 22.990234 6.2457031 22.14 6.8457031 21 L 9.4472656 21 C 11.747266 21 13.447266 19.19 13.447266 17 L 13.447266 15.869141 C 13.768504 15.952624 14.100702 16.009766 14.447266 16.009766 L 17.146484 16.009766 C 17.646484 17.159766 18.746484 18.009766 20.146484 18.009766 C 22.046484 18.009766 23.646484 16.449766 23.646484 14.509766 C 23.646484 12.579766 22.046484 11.009766 20.146484 11.009766 C 18.746484 11.009766 17.646484 11.859766 17.146484 13.009766 L 15.009766 13.009766 C 15.119625 12.684735 15.189453 12.346256 15.189453 12 C 15.189453 11.16 14.849906 10.339953 14.253906 9.7519531 C 14.0189 9.51021 13.74069 9.3244522 13.447266 9.171875 L 13.447266 7.0097656 C 13.447266 6.4597656 13.947266 6.0097656 14.447266 6.0097656 L 17.146484 6.0097656 C 17.646484 7.1597656 18.746484 8.0097656 20.146484 8.0097656 C 22.046484 8.0097656 23.646484 6.4397656 23.646484 4.5097656 C 23.646484 2.5697656 22.046484 1.0097656 20.146484 1.0097656 z M 20.146484 2.0097656 C 21.446484 2.0097656 22.646484 3.1297656 22.646484 4.5097656 C 22.646484 5.8797656 21.446484 7.0097656 20.146484 7.0097656 C 19.046484 7.0097656 18.145703 6.3096094 17.845703 5.3496094 L 17.746094 5.0097656 L 14.447266 5.0097656 C 13.347266 5.0097656 12.447266 5.8997656 12.447266 7.0097656 L 12.447266 8.8476562 C 12.298996 8.8261586 12.150754 8.8027344 12 8.8027344 C 11.954455 8.8027344 11.910576 8.8144662 11.865234 8.8164062 C 11.733157 8.716719 11.592447 8.6297054 11.447266 8.546875 L 11.447266 7.0097656 C 11.447266 5.3597656 12.847266 4.0097656 14.447266 4.0097656 L 17.746094 4.0097656 L 17.845703 3.6699219 C 18.145703 2.7099219 19.046484 2.0097656 20.146484 2.0097656 z M 3.8457031 7.0097656 C 4.9457031 7.0097656 5.8464844 7.7099219 6.1464844 8.6699219 L 6.2460938 9.0097656 L 9.4472656 9.0097656 C 9.8222656 9.0097656 10.165234 9.0792969 10.474609 9.2050781 C 10.207952 9.3508551 9.9554097 9.5233651 9.7402344 9.7421875 C 9.6554755 9.8255337 9.5878282 9.9233484 9.5136719 10.015625 C 9.4909069 10.014746 9.470428 10.009766 9.4472656 10.009766 L 6.2460938 10.009766 L 6.1464844 10.349609 C 5.8464844 11.319609 4.9457031 12.009766 3.8457031 12.009766 C 2.4457031 12.009766 1.3457031 10.899766 1.3457031 9.5097656 C 1.3457031 8.1197656 2.4457031 7.0097656 3.8457031 7.0097656 z M 20.146484 12.009766 C 21.446484 12.009766 22.646484 13.139766 22.646484 14.509766 C 22.646484 15.889766 21.446484 17.009766 20.146484 17.009766 C 19.046484 17.009766 18.145703 16.309609 17.845703 15.349609 L 17.746094 15.009766 L 14.447266 15.009766 C 14.100959 15.009766 13.772729 14.94045 13.470703 14.816406 C 13.754756 14.666178 14.02454 14.485593 14.253906 14.253906 C 14.328913 14.179151 14.386367 14.091269 14.453125 14.009766 L 17.746094 14.009766 L 17.845703 13.669922 C 18.145703 12.709922 19.046484 12.009766 20.146484 12.009766 z M 11.447266 15.144531 C 11.629002 15.17624 11.813246 15.199219 12 15.199219 C 12.018544 15.199153 12.036184 15.193748 12.054688 15.193359 C 12.180437 15.288088 12.3107 15.373496 12.447266 15.453125 L 12.447266 17 C 12.447266 18.67 11.147266 20 9.4472656 20 L 6.6464844 20 L 6.2460938 20 L 6.1464844 20.330078 C 5.8464844 21.290078 4.9457031 21.990234 3.8457031 21.990234 C 2.4457031 21.990234 1.3457031 20.870469 1.3457031 19.480469 C 1.3457031 18.090469 2.4457031 16.980469 3.8457031 16.980469 C 4.9457031 16.980469 5.8464844 17.680625 6.1464844 18.640625 L 6.2460938 18.980469 L 9.4472656 18.980469 C 10.547266 18.980469 11.447266 18.090469 11.447266 16.980469 L 11.447266 15.144531 z " + style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:medium;line-height:normal;font-family:sans-serif;text-indent:0;text-align:start;text-decoration:none;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;letter-spacing:normal;word-spacing:normal;text-transform:none;direction:ltr;block-progression:tb;writing-mode:lr-tb;baseline-shift:baseline;text-anchor:start;white-space:normal;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;fill:#000000;fill-opacity:0.5;fill-rule:nonzero;stroke:none;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate" /> + </g> + </g> + </g> + </g> + <path d="M 3,3 l 18,18" style="stroke-width: 3; stroke: #000000; stroke-linecap: round;" /> + <path d="M 21,3 l -18,18" style="stroke-width: 3; stroke: #000000; stroke-linecap: round;" /> +</svg> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index f10dddf589..520d229901 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -343,7 +343,6 @@ RPCConsole::RPCConsole(const PlatformStyle *_platformStyle, QWidget *parent) : ui(new Ui::RPCConsole), clientModel(0), historyPtr(0), - cachedNodeid(-1), platformStyle(_platformStyle), peersTableContextMenu(0), banTableContextMenu(0), @@ -383,7 +382,6 @@ RPCConsole::RPCConsole(const PlatformStyle *_platformStyle, QWidget *parent) : // based timer interface RPCSetTimerInterfaceIfUnset(rpcTimerInterface); - startExecutor(); setTrafficGraphRange(INITIAL_TRAFFIC_GRAPH_MINS); ui->detailWidget->hide(); @@ -397,7 +395,6 @@ RPCConsole::RPCConsole(const PlatformStyle *_platformStyle, QWidget *parent) : RPCConsole::~RPCConsole() { GUIUtil::saveWindowGeometry("nRPCConsoleWindow", this); - Q_EMIT stopExecutor(); RPCUnsetTimerInterface(rpcTimerInterface); delete rpcTimerInterface; delete ui; @@ -459,6 +456,9 @@ void RPCConsole::setClientModel(ClientModel *model) setNumBlocks(model->getNumBlocks(), model->getLastBlockDate(), model->getVerificationProgress(NULL), false); connect(model, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); + updateNetworkState(); + connect(model, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + updateTrafficStats(model->getTotalBytesRecv(), model->getTotalBytesSent()); connect(model, SIGNAL(bytesChanged(quint64,quint64)), this, SLOT(updateTrafficStats(quint64, quint64))); @@ -469,7 +469,7 @@ void RPCConsole::setClientModel(ClientModel *model) ui->peerWidget->verticalHeader()->hide(); ui->peerWidget->setEditTriggers(QAbstractItemView::NoEditTriggers); ui->peerWidget->setSelectionBehavior(QAbstractItemView::SelectRows); - ui->peerWidget->setSelectionMode(QAbstractItemView::SingleSelection); + ui->peerWidget->setSelectionMode(QAbstractItemView::ExtendedSelection); ui->peerWidget->setContextMenuPolicy(Qt::CustomContextMenu); ui->peerWidget->setColumnWidth(PeerTableModel::Address, ADDRESS_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Subversion, SUBVERSION_COLUMN_WIDTH); @@ -477,14 +477,14 @@ void RPCConsole::setClientModel(ClientModel *model) ui->peerWidget->horizontalHeader()->setStretchLastSection(true); // create peer table context menu actions - QAction* disconnectAction = new QAction(tr("&Disconnect Node"), this); - QAction* banAction1h = new QAction(tr("Ban Node for") + " " + tr("1 &hour"), this); - QAction* banAction24h = new QAction(tr("Ban Node for") + " " + tr("1 &day"), this); - QAction* banAction7d = new QAction(tr("Ban Node for") + " " + tr("1 &week"), this); - QAction* banAction365d = new QAction(tr("Ban Node for") + " " + tr("1 &year"), this); + QAction* disconnectAction = new QAction(tr("&Disconnect"), this); + QAction* banAction1h = new QAction(tr("Ban for") + " " + tr("1 &hour"), this); + QAction* banAction24h = new QAction(tr("Ban for") + " " + tr("1 &day"), this); + QAction* banAction7d = new QAction(tr("Ban for") + " " + tr("1 &week"), this); + QAction* banAction365d = new QAction(tr("Ban for") + " " + tr("1 &year"), this); // create peer table context menu - peersTableContextMenu = new QMenu(); + peersTableContextMenu = new QMenu(this); peersTableContextMenu->addAction(disconnectAction); peersTableContextMenu->addAction(banAction1h); peersTableContextMenu->addAction(banAction24h); @@ -514,7 +514,9 @@ void RPCConsole::setClientModel(ClientModel *model) this, SLOT(peerSelected(const QItemSelection &, const QItemSelection &))); // peer table signal handling - update peer details when new nodes are added to the model connect(model->getPeerTableModel(), SIGNAL(layoutChanged()), this, SLOT(peerLayoutChanged())); - + // peer table signal handling - cache selected node ids + connect(model->getPeerTableModel(), SIGNAL(layoutAboutToChange()), this, SLOT(peerLayoutAboutToChange())); + // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); ui->banlistWidget->verticalHeader()->hide(); @@ -527,10 +529,10 @@ void RPCConsole::setClientModel(ClientModel *model) ui->banlistWidget->horizontalHeader()->setStretchLastSection(true); // create ban table context menu action - QAction* unbanAction = new QAction(tr("&Unban Node"), this); + QAction* unbanAction = new QAction(tr("&Unban"), this); // create ban table context menu - banTableContextMenu = new QMenu(); + banTableContextMenu = new QMenu(this); banTableContextMenu->addAction(unbanAction); // ban table context menu signals @@ -561,6 +563,14 @@ void RPCConsole::setClientModel(ClientModel *model) autoCompleter = new QCompleter(wordList, this); ui->lineEdit->setCompleter(autoCompleter); autoCompleter->popup()->installEventFilter(this); + // Start thread to execute RPC commands. + startExecutor(); + } + if (!model) { + // Client model is being set to 0, this means shutdown() is about to be called. + // Make sure we clean up the executor thread + Q_EMIT stopExecutor(); + thread.wait(); } } @@ -673,16 +683,30 @@ void RPCConsole::message(int category, const QString &message, bool html) ui->messagesWidget->append(out); } +void RPCConsole::updateNetworkState() +{ + QString connections = QString::number(clientModel->getNumConnections()) + " ("; + connections += tr("In:") + " " + QString::number(clientModel->getNumConnections(CONNECTIONS_IN)) + " / "; + connections += tr("Out:") + " " + QString::number(clientModel->getNumConnections(CONNECTIONS_OUT)) + ")"; + + if(!clientModel->getNetworkActive()) { + connections += " (" + tr("Network activity disabled") + ")"; + } + + ui->numberOfConnections->setText(connections); +} + void RPCConsole::setNumConnections(int count) { if (!clientModel) return; - QString connections = QString::number(count) + " ("; - connections += tr("In:") + " " + QString::number(clientModel->getNumConnections(CONNECTIONS_IN)) + " / "; - connections += tr("Out:") + " " + QString::number(clientModel->getNumConnections(CONNECTIONS_OUT)) + ")"; + updateNetworkState(); +} - ui->numberOfConnections->setText(connections); +void RPCConsole::setNetworkActive(bool networkActive) +{ + updateNetworkState(); } void RPCConsole::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers) @@ -741,9 +765,8 @@ void RPCConsole::browseHistory(int offset) void RPCConsole::startExecutor() { - QThread *thread = new QThread; RPCExecutor *executor = new RPCExecutor(); - executor->moveToThread(thread); + executor->moveToThread(&thread); // Replies from executor object must go to this object connect(executor, SIGNAL(reply(int,QString)), this, SLOT(message(int,QString))); @@ -751,16 +774,15 @@ void RPCConsole::startExecutor() connect(this, SIGNAL(cmdRequest(QString)), executor, SLOT(request(QString))); // On stopExecutor signal - // - queue executor for deletion (in execution thread) // - quit the Qt event loop in the execution thread - connect(this, SIGNAL(stopExecutor()), executor, SLOT(deleteLater())); - connect(this, SIGNAL(stopExecutor()), thread, SLOT(quit())); - // Queue the thread for deletion (in this thread) when it is finished - connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); + connect(this, SIGNAL(stopExecutor()), &thread, SLOT(quit())); + // - queue executor for deletion (in execution thread) + connect(&thread, SIGNAL(finished()), executor, SLOT(deleteLater()), Qt::DirectConnection); + connect(&thread, SIGNAL(finished()), this, SLOT(test()), Qt::DirectConnection); // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. - thread->start(); + thread.start(); } void RPCConsole::on_tabWidget_currentChanged(int index) @@ -825,6 +847,17 @@ void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelecti updateNodeDetail(stats); } +void RPCConsole::peerLayoutAboutToChange() +{ + QModelIndexList selected = ui->peerWidget->selectionModel()->selectedIndexes(); + cachedNodeids.clear(); + for(int i = 0; i < selected.size(); i++) + { + const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected.at(i).row()); + cachedNodeids.append(stats->nodeStats.nodeid); + } +} + void RPCConsole::peerLayoutChanged() { if (!clientModel || !clientModel->getPeerTableModel()) @@ -834,7 +867,7 @@ void RPCConsole::peerLayoutChanged() bool fUnselect = false; bool fReselect = false; - if (cachedNodeid == -1) // no node selected yet + if (cachedNodeids.empty()) // no node selected yet return; // find the currently selected row @@ -846,7 +879,7 @@ void RPCConsole::peerLayoutChanged() // check if our detail node has a row in the table (it may not necessarily // be at selectedRow since its position can change after a layout change) - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeid); + int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.first()); if (detailNodeRow < 0) { @@ -872,7 +905,10 @@ void RPCConsole::peerLayoutChanged() if (fReselect) { - ui->peerWidget->selectRow(detailNodeRow); + for(int i = 0; i < cachedNodeids.size(); i++) + { + ui->peerWidget->selectRow(clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.at(i))); + } } if (stats) @@ -881,9 +917,6 @@ void RPCConsole::peerLayoutChanged() void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) { - // Update cached nodeid - cachedNodeid = stats->nodeStats.nodeid; - // update the detail ui with latest node information QString peerAddrDetails(QString::fromStdString(stats->nodeStats.addrName) + " "); peerAddrDetails += tr("(node id: %1)").arg(QString::number(stats->nodeStats.nodeid)); @@ -973,33 +1006,44 @@ void RPCConsole::disconnectSelectedNode() { if(!g_connman) return; - // Get currently selected peer address - NodeId id = GUIUtil::getEntryData(ui->peerWidget, 0, PeerTableModel::NetNodeId).toInt(); - // Find the node, disconnect it and clear the selected node - if(g_connman->DisconnectNode(id)) - clearSelectedNode(); + + // Get selected peer addresses + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + for(int i = 0; i < nodes.count(); i++) + { + // Get currently selected peer address + NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + // Find the node, disconnect it and clear the selected node + if(g_connman->DisconnectNode(id)) + clearSelectedNode(); + } } void RPCConsole::banSelectedNode(int bantime) { if (!clientModel || !g_connman) return; - - if(cachedNodeid == -1) - return; - - // Get currently selected peer address - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeid); - if(detailNodeRow < 0) - return; - - // Find possible nodes, ban it and clear the selected node - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); - if(stats) { - g_connman->Ban(stats->nodeStats.addr, BanReasonManuallyAdded, bantime); - clearSelectedNode(); - clientModel->getBanTableModel()->refresh(); + + // Get selected peer addresses + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, 0); + for(int i = 0; i < nodes.count(); i++) + { + // Get currently selected peer address + NodeId id = nodes.at(i).data(PeerTableModel::NetNodeId).toInt(); + + // Get currently selected peer address + int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); + if(detailNodeRow < 0) + return; + + // Find possible nodes, ban it and clear the selected node + const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); + if(stats) { + g_connman->Ban(stats->nodeStats.addr, BanReasonManuallyAdded, bantime); + } } + clearSelectedNode(); + clientModel->getBanTableModel()->refresh(); } void RPCConsole::unbanSelectedNode() @@ -1007,22 +1051,27 @@ void RPCConsole::unbanSelectedNode() if (!clientModel) return; - // Get currently selected ban address - QString strNode = GUIUtil::getEntryData(ui->banlistWidget, 0, BanTableModel::Address).toString(); - CSubNet possibleSubnet; - - LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); - if (possibleSubnet.IsValid() && g_connman) + // Get selected ban addresses + QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->banlistWidget, 0); + for(int i = 0; i < nodes.count(); i++) { - g_connman->Unban(possibleSubnet); - clientModel->getBanTableModel()->refresh(); + // Get currently selected ban address + QString strNode = nodes.at(i).data(BanTableModel::Address).toString(); + CSubNet possibleSubnet; + + LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); + if (possibleSubnet.IsValid() && g_connman) + { + g_connman->Unban(possibleSubnet); + clientModel->getBanTableModel()->refresh(); + } } } void RPCConsole::clearSelectedNode() { ui->peerWidget->selectionModel()->clearSelection(); - cachedNodeid = -1; + cachedNodeids.clear(); ui->detailWidget->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); } @@ -1041,3 +1090,8 @@ void RPCConsole::setTabFocus(enum TabTypes tabType) { ui->tabWidget->setCurrentIndex(tabType); } + +void RPCConsole::on_toggleNetworkActiveButton_clicked() +{ + clientModel->setNetworkActive(!clientModel->getNetworkActive()); +} diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 50224a1cc0..344d5ecb98 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -12,6 +12,7 @@ #include <QWidget> #include <QCompleter> +#include <QThread> class ClientModel; class PlatformStyle; @@ -61,6 +62,8 @@ protected: private Q_SLOTS: void on_lineEdit_returnPressed(); void on_tabWidget_currentChanged(int index); + /** toggle network activity */ + void on_toggleNetworkActiveButton_clicked(); /** open the debug.log from the current datadir */ void on_openDebugLogfileButton_clicked(); /** change the time range of the network traffic graph */ @@ -88,6 +91,8 @@ public Q_SLOTS: void message(int category, const QString &message, bool html = false); /** Set number of connections shown in the UI */ void setNumConnections(int count); + /** Set network state shown in the UI */ + void setNetworkActive(bool networkActive); /** Set number of blocks and last block date shown in the UI */ void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); /** Set size (number of transactions and memory usage) of the mempool in the UI */ @@ -98,6 +103,8 @@ public Q_SLOTS: void scrollToEnd(); /** Handle selection of peer in peers list */ void peerSelected(const QItemSelection &selected, const QItemSelection &deselected); + /** Handle selection caching before update */ + void peerLayoutAboutToChange(); /** Handle updated peer information */ void peerLayoutChanged(); /** Disconnect a selected node on the Peers tab */ @@ -135,13 +142,17 @@ private: ClientModel *clientModel; QStringList history; int historyPtr; - NodeId cachedNodeid; + QList<NodeId> cachedNodeids; const PlatformStyle *platformStyle; RPCTimerInterface *rpcTimerInterface; QMenu *peersTableContextMenu; QMenu *banTableContextMenu; int consoleFontSize; QCompleter *autoCompleter; + QThread thread; + + /** Update UI with latest network info from model. */ + void updateNetworkState(); }; #endif // BITCOIN_QT_RPCCONSOLE_H diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index f1c867e6f8..57b2179435 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -110,7 +110,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); - ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); @@ -172,6 +171,13 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateMinFeeLabel(); updateSmartFeeLabel(); updateGlobalFeeVariables(); + + // set the smartfee-sliders default value (wallets default conf.target or last stored value) + QSettings settings; + if (settings.value("nSmartFeeSliderPosition").toInt() == 0) + ui->sliderSmartFee->setValue(ui->sliderSmartFee->maximum() - model->getDefaultConfirmTarget() + 1); + else + ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); } } @@ -229,10 +235,17 @@ void SendCoinsDialog::on_sendButton_clicked() // prepare transaction for getting txFee earlier WalletModelTransaction currentTransaction(recipients); WalletModel::SendCoinsReturn prepareStatus; - if (model->getOptionsModel()->getCoinControlFeatures()) // coin control enabled - prepareStatus = model->prepareTransaction(currentTransaction, CoinControlDialog::coinControl); + + // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled + CCoinControl ctrl; + if (model->getOptionsModel()->getCoinControlFeatures()) + ctrl = *CoinControlDialog::coinControl; + if (ui->radioSmartFee->isChecked()) + ctrl.nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; else - prepareStatus = model->prepareTransaction(currentTransaction); + ctrl.nConfirmTarget = 0; + + prepareStatus = model->prepareTransaction(currentTransaction, &ctrl); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, @@ -521,7 +534,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::TransactionCommitFailed: - msgParams.first = tr("The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of wallet.dat and coins were spent in the copy but not marked as spent here."); + msgParams.first = tr("The transaction was rejected with the following reason: %1").arg(sendCoinsReturn.reasonCommitFailed); msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::AbsurdFee: @@ -576,6 +589,7 @@ void SendCoinsDialog::updateFeeSectionControls() ui->labelFeeEstimation ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeNormal ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeFast ->setEnabled(ui->radioSmartFee->isChecked()); + ui->confirmationTargetLabel ->setEnabled(ui->radioSmartFee->isChecked()); ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); @@ -587,15 +601,17 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - nTxConfirmTarget = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; payTxFee = CFeeRate(0); // set nMinimumTotalFee to 0 to not accidentally pay a custom fee CoinControlDialog::coinControl->nMinimumTotalFee = 0; + + // show the estimated reuquired time for confirmation + ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget*600)+" / "+tr("%n block(s)", "", nConfirmTarget)); } else { - nTxConfirmTarget = defaultConfirmTarget; payTxFee = CFeeRate(ui->customFee->value()); // if user has selected to set a minimum absolute fee, pass the value to coincontrol @@ -630,7 +646,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; int estimateFoundAtBlocks = nBlocksToConfirm; CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); if (feeRate <= CFeeRate(0)) // not enough data => minfee @@ -701,6 +717,8 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) if (!checked && model) // coin control features disabled CoinControlDialog::coinControl->SetNull(); + // make sure we set back the confirmation target + updateGlobalFeeVariables(); coinControlUpdateLabels(); } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 83dac0bd11..b0df495a98 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -26,8 +26,6 @@ QT_BEGIN_NAMESPACE class QUrl; QT_END_NAMESPACE -const int defaultConfirmTarget = 25; - /** Dialog for sending bitcoins */ class SendCoinsDialog : public QDialog { diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index cd27385653..1b6e28e93e 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -147,6 +147,7 @@ void SplashScreen::slotFinish(QWidget *mainWin) if (isMinimized()) showNormal(); hide(); + deleteLater(); // No more need for this } static void InitMessage(SplashScreen *splash, const std::string &message) diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 856b16d2c4..f5cbde6238 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -37,7 +37,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), model(0), transactionProxyModel(0), - transactionView(0), abandonAction(0) + transactionView(0), abandonAction(0), columnResizingFixer(0) { // Build filter row setContentsMargins(0,0,0,0); @@ -147,7 +147,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *editLabelAction = new QAction(tr("Edit label"), this); QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); - contextMenu = new QMenu(); + contextMenu = new QMenu(this); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -212,7 +212,7 @@ void TransactionView::setModel(WalletModel *_model) transactionView->setColumnWidth(TransactionTableModel::Type, TYPE_COLUMN_WIDTH); transactionView->setColumnWidth(TransactionTableModel::Amount, AMOUNT_MINIMUM_COLUMN_WIDTH); - columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(transactionView, AMOUNT_MINIMUM_COLUMN_WIDTH, MINIMUM_COLUMN_WIDTH); + columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(transactionView, AMOUNT_MINIMUM_COLUMN_WIDTH, MINIMUM_COLUMN_WIDTH, this); if (_model->getOptionsModel()) { diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 947bcdb15a..4ec022881c 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -171,22 +171,20 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): setLayout(layout); } -void ShutdownWindow::showShutdownWindow(BitcoinGUI *window) +QWidget *ShutdownWindow::showShutdownWindow(BitcoinGUI *window) { if (!window) - return; + return nullptr; // Show a simple window indicating shutdown status QWidget *shutdownWindow = new ShutdownWindow(); - // We don't hold a direct pointer to the shutdown window after creation, so use - // Qt::WA_DeleteOnClose to make sure that the window will be deleted eventually. - shutdownWindow->setAttribute(Qt::WA_DeleteOnClose); shutdownWindow->setWindowTitle(window->windowTitle()); // Center shutdown window at where main window was const QPoint global = window->mapToGlobal(window->rect().center()); shutdownWindow->move(global.x() - shutdownWindow->width() / 2, global.y() - shutdownWindow->height() / 2); shutdownWindow->show(); + return shutdownWindow; } void ShutdownWindow::closeEvent(QCloseEvent *event) diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 843bd7f67b..b930429578 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -43,7 +43,7 @@ class ShutdownWindow : public QWidget public: ShutdownWindow(QWidget *parent=0, Qt::WindowFlags f=0); - static void showShutdownWindow(BitcoinGUI *window); + static QWidget *showShutdownWindow(BitcoinGUI *window); protected: void closeEvent(QCloseEvent *event); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 305cb4fefa..3490d1c1cc 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -5,6 +5,7 @@ #include "walletmodel.h" #include "addresstablemodel.h" +#include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" #include "paymentserver.h" @@ -328,8 +329,9 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran } CReserveKey *keyChange = transaction.getPossibleKeyChange(); - if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get())) - return TransactionCommitFailed; + CValidationState state; + if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get(), state)) + return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(state.GetRejectReason())); CTransaction* t = (CTransaction*)newTx; CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -698,3 +700,8 @@ bool WalletModel::hdEnabled() const { return wallet->IsHDEnabled(); } + +int WalletModel::getDefaultConfirmTarget() const +{ + return nTxConfirmTarget; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index cdac60da36..eedf6e8cea 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -65,7 +65,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { std::string sAddress = address.toStdString(); std::string sLabel = label.toStdString(); std::string sMessage = message.toStdString(); @@ -144,9 +144,13 @@ public: // Return status record for SendCoins, contains error id + information struct SendCoinsReturn { - SendCoinsReturn(StatusCode _status = OK): - status(_status) {} + SendCoinsReturn(StatusCode _status = OK, QString _reasonCommitFailed = "") + : status(_status), + reasonCommitFailed(_reasonCommitFailed) + { + } StatusCode status; + QString reasonCommitFailed; }; // prepare transaction for getting txfee before sending coins @@ -207,6 +211,8 @@ public: bool hdEnabled() const; + int getDefaultConfirmTarget() const; + private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/rest.cpp b/src/rest.cpp index b8b5420626..90cca6f480 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -50,7 +50,7 @@ struct CCoin { ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nTxVer); READWRITE(nHeight); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7c128885da..733d14d24e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -119,16 +119,16 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx result.push_back(Pair("versionHex", strprintf("%08x", block.nVersion))); result.push_back(Pair("merkleroot", block.hashMerkleRoot.GetHex())); UniValue txs(UniValue::VARR); - BOOST_FOREACH(const CTransaction&tx, block.vtx) + for(const auto& tx : block.vtx) { if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(tx, uint256(), objTx); + TxToJSON(*tx, uint256(), objTx); txs.push_back(objTx); } else - txs.push_back(tx.GetHash().GetHex()); + txs.push_back(tx->GetHash().GetHex()); } result.push_back(Pair("tx", txs)); result.push_back(Pair("time", block.GetBlockTime())); @@ -194,12 +194,12 @@ UniValue waitfornewblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) throw runtime_error( - "waitfornewblock\n" + "waitfornewblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" "\nArguments:\n" - "1. timeout (milliseconds) (int, optional, default=false)\n" - "\nResult::\n" + "1. timeout (int, optional, default=0) Time in milliseconds to wait for a response. 0 indicates no timeout.\n" + "\nResult:\n" "{ (json object)\n" " \"hash\" : { (string) The blockhash\n" " \"height\" : { (int) Block height\n" @@ -232,13 +232,13 @@ UniValue waitforblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw runtime_error( - "waitforblock\n" + "waitforblock <blockhash> (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" "\nArguments:\n" "1. blockhash to wait for (string)\n" - "2. timeout (milliseconds) (int, optional, default=false)\n" - "\nResult::\n" + "2. timeout (int, optional, default=0) Time in milliseconds to wait for a response. 0 indicates no timeout.\n" + "\nResult:\n" "{ (json object)\n" " \"hash\" : { (string) The blockhash\n" " \"height\" : { (int) Block height\n" @@ -274,14 +274,14 @@ UniValue waitforblockheight(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw runtime_error( - "waitforblock\n" + "waitforblockheight <blockheight> (timeout)\n" "\nWaits for (at least) block height and returns the height and hash\n" - "\nof the current tip.\n" + "of the current tip.\n" "\nReturns the current block on timeout or exit.\n" "\nArguments:\n" "1. block height to wait for (int)\n" - "2. timeout (milliseconds) (int, optional, default=false)\n" - "\nResult::\n" + "2. timeout (int, optional, default=0) Time in milliseconds to wait for a response. 0 indicates no timeout.\n" + "\nResult:\n" "{ (json object)\n" " \"hash\" : { (string) The blockhash\n" " \"height\" : { (int) Block height\n" @@ -863,7 +863,7 @@ UniValue gettxout(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" "2. n (numeric, required) vout number\n" - "3. includemempool (boolean, optional) Whether to include the mem pool\n" + "3. includemempool (boolean, optional) Whether to include the mempool\n" "\nResult:\n" "{\n" " \"bestblock\" : \"hash\", (string) the block hash\n" diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 8370a0f43e..03992a278d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -109,6 +109,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "prioritisetransaction", 2 }, { "setban", 2 }, { "setban", 3 }, + { "setnetworkactive", 0 }, { "getmempoolancestors", 1 }, { "getmempooldescendants", 1 }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d509dd691f..f3cd1fbf0b 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -131,8 +131,7 @@ UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nG if (pblock->nNonce == nInnerLoopCount) { continue; } - CValidationState state; - if (!ProcessNewBlock(state, Params(), NULL, pblock, true, NULL)) + if (!ProcessNewBlock(Params(), pblock, true, NULL, NULL)) throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); ++nHeight; blockHashes.push_back(pblock->GetHash().GetHex()); @@ -230,7 +229,7 @@ UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"errors\": \"...\" (string) Current errors\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mem pool\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" "}\n" "\nExamples:\n" @@ -558,7 +557,8 @@ UniValue getblocktemplate(const JSONRPCRequest& request) UniValue transactions(UniValue::VARR); map<uint256, int64_t> setTxIndex; int i = 0; - BOOST_FOREACH (CTransaction& tx, pblock->vtx) { + for (const auto& it : pblock->vtx) { + const CTransaction& tx = *it; uint256 txHash = tx.GetHash(); setTxIndex[txHash] = i++; @@ -663,7 +663,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) result.push_back(Pair("previousblockhash", pblock->hashPrevBlock.GetHex())); result.push_back(Pair("transactions", transactions)); result.push_back(Pair("coinbaseaux", aux)); - result.push_back(Pair("coinbasevalue", (int64_t)pblock->vtx[0].vout[0].nValue)); + result.push_back(Pair("coinbasevalue", (int64_t)pblock->vtx[0]->vout[0].nValue)); result.push_back(Pair("longpollid", chainActive.Tip()->GetBlockHash().GetHex() + i64tostr(nTransactionsUpdatedLast))); result.push_back(Pair("target", hashTarget.GetHex())); result.push_back(Pair("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1)); @@ -680,7 +680,9 @@ UniValue getblocktemplate(const JSONRPCRequest& request) result.push_back(Pair("curtime", pblock->GetBlockTime())); result.push_back(Pair("bits", strprintf("%08x", pblock->nBits))); result.push_back(Pair("height", (int64_t)(pindexPrev->nHeight+1))); - if (!pblocktemplate->vchCoinbaseCommitment.empty()) { + + const struct BIP9DeploymentInfo& segwit_info = VersionBitsDeploymentInfo[Consensus::DEPLOYMENT_SEGWIT]; + if (!pblocktemplate->vchCoinbaseCommitment.empty() && setClientRules.find(segwit_info.name) != setClientRules.end()) { result.push_back(Pair("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment.begin(), pblocktemplate->vchCoinbaseCommitment.end()))); } @@ -754,10 +756,9 @@ UniValue submitblock(const JSONRPCRequest& request) } } - CValidationState state; submitblock_StateCatcher sc(block.GetHash()); RegisterValidationInterface(&sc); - bool fAccepted = ProcessNewBlock(state, Params(), NULL, &block, true, NULL); + bool fAccepted = ProcessNewBlock(Params(), &block, true, NULL, NULL); UnregisterValidationInterface(&sc); if (fBlockPresent) { @@ -765,13 +766,9 @@ UniValue submitblock(const JSONRPCRequest& request) return "duplicate-inconclusive"; return "duplicate"; } - if (fAccepted) - { - if (!sc.found) - return "inconclusive"; - state = sc.state; - } - return BIP22ValidationResult(state); + if (!sc.found) + return "inconclusive"; + return BIP22ValidationResult(sc.state); } UniValue estimatefee(const JSONRPCRequest& request) @@ -810,7 +807,7 @@ UniValue estimatepriority(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw runtime_error( "estimatepriority nblocks\n" - "\nEstimates the approximate priority a zero-fee transaction needs to begin\n" + "\nDEPRECATED. Estimates the approximate priority a zero-fee transaction needs to begin\n" "confirmation within nblocks blocks.\n" "\nArguments:\n" "1. nblocks (numeric)\n" @@ -873,7 +870,7 @@ UniValue estimatesmartpriority(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw runtime_error( "estimatesmartpriority nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nDEPRECATED. WARNING: This interface is unstable and may disappear or change!\n" "\nEstimates the approximate priority a zero-fee transaction needs to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid.\n" diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index eaef4856b3..3193985803 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -57,7 +57,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"testnet\": true|false, (boolean) if the server is using testnet or not\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" @@ -450,10 +450,53 @@ UniValue setmocktime(const JSONRPCRequest& request) return NullUniValue; } +static UniValue RPCLockedMemoryInfo() +{ + LockedPool::Stats stats = LockedPoolManager::Instance().stats(); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("used", uint64_t(stats.used))); + obj.push_back(Pair("free", uint64_t(stats.free))); + obj.push_back(Pair("total", uint64_t(stats.total))); + obj.push_back(Pair("locked", uint64_t(stats.locked))); + obj.push_back(Pair("chunks_used", uint64_t(stats.chunks_used))); + obj.push_back(Pair("chunks_free", uint64_t(stats.chunks_free))); + return obj; +} + +UniValue getmemoryinfo(const JSONRPCRequest& request) +{ + /* Please, avoid using the word "pool" here in the RPC interface or help, + * as users will undoubtedly confuse it with the other "memory pool" + */ + if (request.fHelp || request.params.size() != 0) + throw runtime_error( + "getmemoryinfo\n" + "Returns an object containing information about memory usage.\n" + "\nResult:\n" + "{\n" + " \"locked\": { (json object) Information about locked memory manager\n" + " \"used\": xxxxx, (numeric) Number of bytes used\n" + " \"free\": xxxxx, (numeric) Number of bytes available in current arenas\n" + " \"total\": xxxxxxx, (numeric) Total number of bytes managed\n" + " \"locked\": xxxxxx, (numeric) Amount of bytes that succeeded locking. If this number is smaller than total, locking pages failed at some point and key data could be swapped to disk.\n" + " \"chunks_used\": xxxxx, (numeric) Number allocated chunks\n" + " \"chunks_free\": xxxxx, (numeric) Number unused chunks\n" + " }\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getmemoryinfo", "") + + HelpExampleRpc("getmemoryinfo", "") + ); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("locked", RPCLockedMemoryInfo())); + return obj; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "control", "getinfo", &getinfo, true }, /* uses wallet if enabled */ + { "control", "getmemoryinfo", &getmemoryinfo, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, true }, { "util", "verifymessage", &verifymessage, true }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 2b43f08f0b..f57ba76d3a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -331,7 +331,7 @@ UniValue getnettotals(const JSONRPCRequest& request) "{\n" " \"totalbytesrecv\": n, (numeric) Total bytes received\n" " \"totalbytessent\": n, (numeric) Total bytes sent\n" - " \"timemillis\": t, (numeric) Total cpu time\n" + " \"timemillis\": t, (numeric) Current UNIX time in milliseconds\n" " \"uploadtarget\":\n" " {\n" " \"timeframe\": n, (numeric) Length of the measuring timeframe in seconds\n" @@ -401,6 +401,7 @@ UniValue getnetworkinfo(const JSONRPCRequest& request) " \"localrelay\": true|false, (bool) true if transaction relay is requested from peers\n" " \"timeoffset\": xxxxx, (numeric) the time offset\n" " \"connections\": xxxxx, (numeric) the number of connections\n" + " \"networkactive\": true|false, (bool) whether p2p networking is enabled\n" " \"networks\": [ (array) information per network\n" " {\n" " \"name\": \"xxx\", (string) network (ipv4, ipv6 or onion)\n" @@ -435,8 +436,10 @@ UniValue getnetworkinfo(const JSONRPCRequest& request) obj.push_back(Pair("localservices", strprintf("%016x", g_connman->GetLocalServices()))); obj.push_back(Pair("localrelay", fRelayTxes)); obj.push_back(Pair("timeoffset", GetTimeOffset())); - if(g_connman) + if (g_connman) { + obj.push_back(Pair("networkactive", g_connman->GetNetworkActive())); obj.push_back(Pair("connections", (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL))); + } obj.push_back(Pair("networks", GetNetworksInfo())); obj.push_back(Pair("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK()))); UniValue localAddresses(UniValue::VARR); @@ -571,6 +574,24 @@ UniValue clearbanned(const JSONRPCRequest& request) return NullUniValue; } +UniValue setnetworkactive(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw runtime_error( + "setnetworkactive true|false\n" + "Disable/enable all p2p network activity." + ); + } + + if (!g_connman) { + throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } + + g_connman->SetNetworkActive(request.params[0].get_bool()); + + return g_connman->GetNetworkActive(); +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -585,6 +606,7 @@ static const CRPCCommand commands[] = { "network", "setban", &setban, true }, { "network", "listbanned", &listbanned, true }, { "network", "clearbanned", &clearbanned, true }, + { "network", "setnetworkactive", &setnetworkactive, true, }, }; void RegisterNetRPCCommands(CRPCTable &t) diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 0656a61755..370c021ea6 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -135,17 +135,17 @@ UniValue getrawtransaction(const JSONRPCRequest& request) "or there is an unspent output in the utxo for this transaction. To make it always work,\n" "you need to maintain a transaction index, using the -txindex command line option.\n" "\nReturn the raw transaction data.\n" - "\nIf verbose=0, returns a string that is serialized, hex-encoded data for 'txid'.\n" - "If verbose is non-zero, returns an Object with information about 'txid'.\n" + "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" + "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.\n" "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (numeric, optional, default=0) If 0, return a string, other return a json object\n" + "2. verbose (bool, optional, default=false) If true, return a string, other return a json object\n" - "\nResult (if verbose is not set or set to 0):\n" + "\nResult (if verbose is not set or set to false):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" - "\nResult (if verbose > 0):\n" + "\nResult (if verbose is set to true):\n" "{\n" " \"hex\" : \"data\", (string) The serialized, hex-encoded data for 'txid'\n" " \"txid\" : \"id\", (string) The transaction id (same as provided)\n" @@ -192,17 +192,31 @@ UniValue getrawtransaction(const JSONRPCRequest& request) "\nExamples:\n" + HelpExampleCli("getrawtransaction", "\"mytxid\"") - + HelpExampleCli("getrawtransaction", "\"mytxid\" 1") - + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") + + HelpExampleCli("getrawtransaction", "\"mytxid\" true") + + HelpExampleRpc("getrawtransaction", "\"mytxid\", true") ); LOCK(cs_main); uint256 hash = ParseHashV(request.params[0], "parameter 1"); + // Accept either a bool (true) or a num (>=1) to indicate verbose output. bool fVerbose = false; - if (request.params.size() > 1) - fVerbose = (request.params[1].get_int() != 0); + if (request.params.size() > 1) { + if (request.params[1].isNum()) { + if (request.params[1].get_int() != 0) { + fVerbose = true; + } + } + else if(request.params[1].isBool()) { + if(request.params[1].isTrue()) { + fVerbose = true; + } + } + else { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid type provided. Verbose parameter must be a boolean."); + } + } CTransaction tx; uint256 hashBlock; @@ -288,8 +302,8 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); unsigned int ntxFound = 0; - BOOST_FOREACH(const CTransaction&tx, block.vtx) - if (setTxids.count(tx.GetHash())) + for (const auto& tx : block.vtx) + if (setTxids.count(tx->GetHash())) ntxFound++; if (ntxFound != setTxids.size()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "(Not all) transactions not found in specified block"); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 29d0bee1b2..164e0f00e2 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -147,6 +147,8 @@ uint256 ParseHashV(const UniValue& v, string strName) strHex = v.get_str(); if (!IsHex(strHex)) // Note: IsHex("") is false throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + if (64 != strHex.length()) + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d)", strName, 64, strHex.length())); uint256 result; result.SetHex(strHex); return result; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 52777b61f9..27c03f154c 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -54,9 +54,10 @@ void CScheduler::serviceQueue() #else // Some boost versions have a conflicting overload of wait_until that returns void. // Explicitly use a template here to avoid hitting that overload. - while (!shouldStop() && !taskQueue.empty() && - newTaskScheduled.wait_until<>(lock, taskQueue.begin()->first) != boost::cv_status::timeout) { - // Keep waiting until timeout + while (!shouldStop() && !taskQueue.empty()) { + boost::chrono::system_clock::time_point timeToWaitFor = taskQueue.begin()->first; + if (newTaskScheduled.wait_until<>(lock, timeToWaitFor) == boost::cv_status::timeout) + break; // Exit loop after timeout, it means we reached the time of the event } #endif // If there are multiple threads, the queue can empty while we're waiting (another diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index b629f4278b..069ac55bfb 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -23,7 +23,7 @@ public: m_remaining(txToLen) {} - TxInputStream& read(char* pch, size_t nSize) + void read(char* pch, size_t nSize) { if (nSize > m_remaining) throw std::ios_base::failure(std::string(__func__) + ": end of data"); @@ -37,16 +37,17 @@ public: memcpy(pch, m_data, nSize); m_remaining -= nSize; m_data += nSize; - return *this; } template<typename T> TxInputStream& operator>>(T& obj) { - ::Unserialize(*this, obj, m_type, m_version); + ::Unserialize(*this, obj); return *this; } + int GetVersion() const { return m_version; } + int GetType() const { return m_type; } private: const int m_type; const int m_version; @@ -69,17 +70,26 @@ struct ECCryptoClosure ECCryptoClosure instance_of_eccryptoclosure; } +/** Check that all specified flags are part of the libconsensus interface. */ +static bool verify_flags(unsigned int flags) +{ + return (flags & ~(bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL)) == 0; +} + static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, CAmount amount, const unsigned char *txTo , unsigned int txToLen, unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) { + if (!verify_flags(flags)) { + return bitcoinconsensus_ERR_INVALID_FLAGS; + } try { TxInputStream stream(SER_NETWORK, PROTOCOL_VERSION, txTo, txToLen); CTransaction tx; stream >> tx; if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (tx.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION) != txToLen) + if (GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) != txToLen) return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); // Regardless of the verification result, the tx did not error. diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 1d2d5c23e4..1bef4fe9e9 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -42,6 +42,7 @@ typedef enum bitcoinconsensus_error_t bitcoinconsensus_ERR_TX_SIZE_MISMATCH, bitcoinconsensus_ERR_TX_DESERIALIZE, bitcoinconsensus_ERR_AMOUNT_REQUIRED, + bitcoinconsensus_ERR_INVALID_FLAGS, } bitcoinconsensus_error; /** Script verification flags */ @@ -54,6 +55,9 @@ enum bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY = (1U << 10), // enable CHECKSEQUENCEVERIFY (BIP112) bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS = (1U << 11), // enable WITNESS (BIP141) + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG | + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY | + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS }; /// Returns 1 if the input nIn of the serialized transaction pointed to by diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 0e17ddc130..a6403f9363 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1069,7 +1069,7 @@ public: /** Serialize the passed scriptCode, skipping OP_CODESEPARATORs */ template<typename S> - void SerializeScriptCode(S &s, int nType, int nVersion) const { + void SerializeScriptCode(S &s) const { CScript::const_iterator it = scriptCode.begin(); CScript::const_iterator itBegin = it; opcodetype opcode; @@ -1092,53 +1092,53 @@ public: /** Serialize an input of txTo */ template<typename S> - void SerializeInput(S &s, unsigned int nInput, int nType, int nVersion) const { + void SerializeInput(S &s, unsigned int nInput) const { // In case of SIGHASH_ANYONECANPAY, only the input being signed is serialized if (fAnyoneCanPay) nInput = nIn; // Serialize the prevout - ::Serialize(s, txTo.vin[nInput].prevout, nType, nVersion); + ::Serialize(s, txTo.vin[nInput].prevout); // Serialize the script if (nInput != nIn) // Blank out other inputs' signatures - ::Serialize(s, CScriptBase(), nType, nVersion); + ::Serialize(s, CScriptBase()); else - SerializeScriptCode(s, nType, nVersion); + SerializeScriptCode(s); // Serialize the nSequence if (nInput != nIn && (fHashSingle || fHashNone)) // let the others update at will - ::Serialize(s, (int)0, nType, nVersion); + ::Serialize(s, (int)0); else - ::Serialize(s, txTo.vin[nInput].nSequence, nType, nVersion); + ::Serialize(s, txTo.vin[nInput].nSequence); } /** Serialize an output of txTo */ template<typename S> - void SerializeOutput(S &s, unsigned int nOutput, int nType, int nVersion) const { + void SerializeOutput(S &s, unsigned int nOutput) const { if (fHashSingle && nOutput != nIn) // Do not lock-in the txout payee at other indices as txin - ::Serialize(s, CTxOut(), nType, nVersion); + ::Serialize(s, CTxOut()); else - ::Serialize(s, txTo.vout[nOutput], nType, nVersion); + ::Serialize(s, txTo.vout[nOutput]); } /** Serialize txTo */ template<typename S> - void Serialize(S &s, int nType, int nVersion) const { + void Serialize(S &s) const { // Serialize nVersion - ::Serialize(s, txTo.nVersion, nType, nVersion); + ::Serialize(s, txTo.nVersion); // Serialize vin unsigned int nInputs = fAnyoneCanPay ? 1 : txTo.vin.size(); ::WriteCompactSize(s, nInputs); for (unsigned int nInput = 0; nInput < nInputs; nInput++) - SerializeInput(s, nInput, nType, nVersion); + SerializeInput(s, nInput); // Serialize vout unsigned int nOutputs = fHashNone ? 0 : (fHashSingle ? nIn+1 : txTo.vout.size()); ::WriteCompactSize(s, nOutputs); for (unsigned int nOutput = 0; nOutput < nOutputs; nOutput++) - SerializeOutput(s, nOutput, nType, nVersion); + SerializeOutput(s, nOutput); // Serialize nLockTime - ::Serialize(s, txTo.nLockTime, nType, nVersion); + ::Serialize(s, txTo.nLockTime); } }; diff --git a/src/serialize.h b/src/serialize.h index 1f51da82ff..e28ca548c0 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -13,6 +13,7 @@ #include <ios> #include <limits> #include <map> +#include <memory> #include <set> #include <stdint.h> #include <string> @@ -25,6 +26,20 @@ static const unsigned int MAX_SIZE = 0x02000000; /** + * Dummy data type to identify deserializing constructors. + * + * By convention, a constructor of a type T with signature + * + * template <typename Stream> T::T(deserialize_type, Stream& s) + * + * is a deserializing constructor, which builds the type by + * deserializing it from s. If T contains const fields, this + * is likely the only way to do so. + */ +struct deserialize_type {}; +constexpr deserialize_type deserialize {}; + +/** * Used to bypass the rule against non-const reference to temporary * where it makes sense with wrappers such as CFlatData or CTxDB */ @@ -151,6 +166,8 @@ inline float ser_uint32_to_float(uint32_t y) // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t) // +class CSizeComputer; + enum { // primary actions @@ -159,7 +176,8 @@ enum SER_GETHASH = (1 << 2), }; -#define READWRITE(obj) (::SerReadWrite(s, (obj), nType, nVersion, ser_action)) +#define READWRITE(obj) (::SerReadWrite(s, (obj), ser_action)) +#define READWRITEMANY(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) /** * Implement three methods for serializable objects. These are actually wrappers over @@ -167,63 +185,42 @@ enum * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be * added as members. */ -#define ADD_SERIALIZE_METHODS \ - size_t GetSerializeSize(int nType, int nVersion) const { \ - CSizeComputer s(nType, nVersion); \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize(), nType, nVersion);\ - return s.size(); \ - } \ - template<typename Stream> \ - void Serialize(Stream& s, int nType, int nVersion) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize(), nType, nVersion);\ - } \ - template<typename Stream> \ - void Unserialize(Stream& s, int nType, int nVersion) { \ - SerializationOp(s, CSerActionUnserialize(), nType, nVersion); \ +#define ADD_SERIALIZE_METHODS \ + template<typename Stream> \ + void Serialize(Stream& s) const { \ + NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ + } \ + template<typename Stream> \ + void Unserialize(Stream& s) { \ + SerializationOp(s, CSerActionUnserialize()); \ } -/* - * Basic Types - */ -inline unsigned int GetSerializeSize(char a, int, int=0) { return 1; } -inline unsigned int GetSerializeSize(int8_t a, int, int=0) { return 1; } -inline unsigned int GetSerializeSize(uint8_t a, int, int=0) { return 1; } -inline unsigned int GetSerializeSize(int16_t a, int, int=0) { return 2; } -inline unsigned int GetSerializeSize(uint16_t a, int, int=0) { return 2; } -inline unsigned int GetSerializeSize(int32_t a, int, int=0) { return 4; } -inline unsigned int GetSerializeSize(uint32_t a, int, int=0) { return 4; } -inline unsigned int GetSerializeSize(int64_t a, int, int=0) { return 8; } -inline unsigned int GetSerializeSize(uint64_t a, int, int=0) { return 8; } -inline unsigned int GetSerializeSize(float a, int, int=0) { return 4; } -inline unsigned int GetSerializeSize(double a, int, int=0) { return 8; } - -template<typename Stream> inline void Serialize(Stream& s, char a, int, int=0) { ser_writedata8(s, a); } // TODO Get rid of bare char -template<typename Stream> inline void Serialize(Stream& s, int8_t a, int, int=0) { ser_writedata8(s, a); } -template<typename Stream> inline void Serialize(Stream& s, uint8_t a, int, int=0) { ser_writedata8(s, a); } -template<typename Stream> inline void Serialize(Stream& s, int16_t a, int, int=0) { ser_writedata16(s, a); } -template<typename Stream> inline void Serialize(Stream& s, uint16_t a, int, int=0) { ser_writedata16(s, a); } -template<typename Stream> inline void Serialize(Stream& s, int32_t a, int, int=0) { ser_writedata32(s, a); } -template<typename Stream> inline void Serialize(Stream& s, uint32_t a, int, int=0) { ser_writedata32(s, a); } -template<typename Stream> inline void Serialize(Stream& s, int64_t a, int, int=0) { ser_writedata64(s, a); } -template<typename Stream> inline void Serialize(Stream& s, uint64_t a, int, int=0) { ser_writedata64(s, a); } -template<typename Stream> inline void Serialize(Stream& s, float a, int, int=0) { ser_writedata32(s, ser_float_to_uint32(a)); } -template<typename Stream> inline void Serialize(Stream& s, double a, int, int=0) { ser_writedata64(s, ser_double_to_uint64(a)); } - -template<typename Stream> inline void Unserialize(Stream& s, char& a, int, int=0) { a = ser_readdata8(s); } // TODO Get rid of bare char -template<typename Stream> inline void Unserialize(Stream& s, int8_t& a, int, int=0) { a = ser_readdata8(s); } -template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a, int, int=0) { a = ser_readdata8(s); } -template<typename Stream> inline void Unserialize(Stream& s, int16_t& a, int, int=0) { a = ser_readdata16(s); } -template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a, int, int=0) { a = ser_readdata16(s); } -template<typename Stream> inline void Unserialize(Stream& s, int32_t& a, int, int=0) { a = ser_readdata32(s); } -template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a, int, int=0) { a = ser_readdata32(s); } -template<typename Stream> inline void Unserialize(Stream& s, int64_t& a, int, int=0) { a = ser_readdata64(s); } -template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a, int, int=0) { a = ser_readdata64(s); } -template<typename Stream> inline void Unserialize(Stream& s, float& a, int, int=0) { a = ser_uint32_to_float(ser_readdata32(s)); } -template<typename Stream> inline void Unserialize(Stream& s, double& a, int, int=0) { a = ser_uint64_to_double(ser_readdata64(s)); } - -inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); } -template<typename Stream> inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; ser_writedata8(s, f); } -template<typename Stream> inline void Unserialize(Stream& s, bool& a, int, int=0) { char f=ser_readdata8(s); a=f; } +template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char +template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); } +template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); } +template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); } +template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); } +template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); } +template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } +template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); } +template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } +template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); } +template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } + +template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char +template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); } +template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); } +template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); } +template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); } +template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); } +template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } +template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); } +template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } +template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); } +template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } + +template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); } +template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; } @@ -245,6 +242,8 @@ inline unsigned int GetSizeOfCompactSize(uint64_t nSize) else return sizeof(unsigned char) + sizeof(uint64_t); } +inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize); + template<typename Stream> void WriteCompactSize(Stream& os, uint64_t nSize) { @@ -339,6 +338,9 @@ inline unsigned int GetSizeOfVarInt(I n) return nRet; } +template<typename I> +inline void WriteVarInt(CSizeComputer& os, I n); + template<typename Stream, typename I> void WriteVarInt(Stream& os, I n) { @@ -402,19 +404,14 @@ public: char* end() { return pend; } const char* end() const { return pend; } - unsigned int GetSerializeSize(int, int=0) const - { - return pend - pbegin; - } - template<typename Stream> - void Serialize(Stream& s, int, int=0) const + void Serialize(Stream& s) const { s.write(pbegin, pend - pbegin); } template<typename Stream> - void Unserialize(Stream& s, int, int=0) + void Unserialize(Stream& s) { s.read(pbegin, pend - pbegin); } @@ -428,17 +425,13 @@ protected: public: CVarInt(I& nIn) : n(nIn) { } - unsigned int GetSerializeSize(int, int) const { - return GetSizeOfVarInt<I>(n); - } - template<typename Stream> - void Serialize(Stream &s, int, int) const { + void Serialize(Stream &s) const { WriteVarInt<Stream,I>(s, n); } template<typename Stream> - void Unserialize(Stream& s, int, int) { + void Unserialize(Stream& s) { n = ReadVarInt<Stream,I>(s); } }; @@ -450,17 +443,13 @@ protected: public: CCompactSize(uint64_t& nIn) : n(nIn) { } - unsigned int GetSerializeSize(int, int) const { - return GetSizeOfCompactSize(n); - } - template<typename Stream> - void Serialize(Stream &s, int, int) const { + void Serialize(Stream &s) const { WriteCompactSize<Stream>(s, n); } template<typename Stream> - void Unserialize(Stream& s, int, int) { + void Unserialize(Stream& s) { n = ReadCompactSize<Stream>(s); } }; @@ -471,10 +460,10 @@ class LimitedString protected: std::string& string; public: - LimitedString(std::string& string) : string(string) {} + LimitedString(std::string& _string) : string(_string) {} template<typename Stream> - void Unserialize(Stream& s, int, int=0) + void Unserialize(Stream& s) { size_t size = ReadCompactSize(s); if (size > Limit) { @@ -486,17 +475,12 @@ public: } template<typename Stream> - void Serialize(Stream& s, int, int=0) const + void Serialize(Stream& s) const { WriteCompactSize(s, string.size()); if (!string.empty()) s.write((char*)&string[0], string.size()); } - - unsigned int GetSerializeSize(int, int=0) const - { - return GetSizeOfCompactSize(string.size()) + string.size(); - } }; template<typename I> @@ -509,85 +493,76 @@ CVarInt<I> WrapVarInt(I& n) { return CVarInt<I>(n); } /** * string */ -template<typename C> unsigned int GetSerializeSize(const std::basic_string<C>& str, int, int=0); -template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str, int, int=0); -template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str, int, int=0); +template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str); +template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str); /** * prevector * prevectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. */ -template<unsigned int N, typename T> unsigned int GetSerializeSize_impl(const prevector<N, T>& v, int nType, int nVersion, const unsigned char&); -template<unsigned int N, typename T, typename V> unsigned int GetSerializeSize_impl(const prevector<N, T>& v, int nType, int nVersion, const V&); -template<unsigned int N, typename T> inline unsigned int GetSerializeSize(const prevector<N, T>& v, int nType, int nVersion); -template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, int nType, int nVersion, const unsigned char&); -template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, int nType, int nVersion, const V&); -template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v, int nType, int nVersion); -template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, const unsigned char&); -template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, const V&); -template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v, int nType, int nVersion); +template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&); +template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&); +template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v); +template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&); +template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&); +template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v); /** * vector * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. */ -template<typename T, typename A> unsigned int GetSerializeSize_impl(const std::vector<T, A>& v, int nType, int nVersion, const unsigned char&); -template<typename T, typename A, typename V> unsigned int GetSerializeSize_impl(const std::vector<T, A>& v, int nType, int nVersion, const V&); -template<typename T, typename A> inline unsigned int GetSerializeSize(const std::vector<T, A>& v, int nType, int nVersion); -template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, int nType, int nVersion, const unsigned char&); -template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, int nType, int nVersion, const V&); -template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v, int nType, int nVersion); -template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, const unsigned char&); -template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, const V&); -template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v, int nType, int nVersion); +template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&); +template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&); +template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v); +template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&); +template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&); +template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v); /** * pair */ -template<typename K, typename T> unsigned int GetSerializeSize(const std::pair<K, T>& item, int nType, int nVersion); -template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item, int nType, int nVersion); -template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item, int nType, int nVersion); +template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item); +template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item); /** * map */ -template<typename K, typename T, typename Pred, typename A> unsigned int GetSerializeSize(const std::map<K, T, Pred, A>& m, int nType, int nVersion); -template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m, int nType, int nVersion); -template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m, int nType, int nVersion); +template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m); +template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m); /** * set */ -template<typename K, typename Pred, typename A> unsigned int GetSerializeSize(const std::set<K, Pred, A>& m, int nType, int nVersion); -template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m, int nType, int nVersion); -template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m, int nType, int nVersion); +template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m); +template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m); +/** + * shared_ptr + */ +template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p); +template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p); +/** + * unique_ptr + */ +template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p); +template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p); /** * If none of the specialized versions above matched, default to calling member function. - * "int nType" is changed to "long nType" to keep from getting an ambiguous overload error. - * The compiler will only cast int to long if none of the other templates matched. - * Thanks to Boost serialization for this idea. */ -template<typename T> -inline unsigned int GetSerializeSize(const T& a, long nType, int nVersion) -{ - return a.GetSerializeSize((int)nType, nVersion); -} - template<typename Stream, typename T> -inline void Serialize(Stream& os, const T& a, long nType, int nVersion) +inline void Serialize(Stream& os, const T& a) { - a.Serialize(os, (int)nType, nVersion); + a.Serialize(os); } template<typename Stream, typename T> -inline void Unserialize(Stream& is, T& a, long nType, int nVersion) +inline void Unserialize(Stream& is, T& a) { - a.Unserialize(is, (int)nType, nVersion); + a.Unserialize(is); } @@ -597,14 +572,8 @@ inline void Unserialize(Stream& is, T& a, long nType, int nVersion) /** * string */ -template<typename C> -unsigned int GetSerializeSize(const std::basic_string<C>& str, int, int) -{ - return GetSizeOfCompactSize(str.size()) + str.size() * sizeof(str[0]); -} - template<typename Stream, typename C> -void Serialize(Stream& os, const std::basic_string<C>& str, int, int) +void Serialize(Stream& os, const std::basic_string<C>& str) { WriteCompactSize(os, str.size()); if (!str.empty()) @@ -612,7 +581,7 @@ void Serialize(Stream& os, const std::basic_string<C>& str, int, int) } template<typename Stream, typename C> -void Unserialize(Stream& is, std::basic_string<C>& str, int, int) +void Unserialize(Stream& is, std::basic_string<C>& str) { unsigned int nSize = ReadCompactSize(is); str.resize(nSize); @@ -625,30 +594,8 @@ void Unserialize(Stream& is, std::basic_string<C>& str, int, int) /** * prevector */ -template<unsigned int N, typename T> -unsigned int GetSerializeSize_impl(const prevector<N, T>& v, int nType, int nVersion, const unsigned char&) -{ - return (GetSizeOfCompactSize(v.size()) + v.size() * sizeof(T)); -} - -template<unsigned int N, typename T, typename V> -unsigned int GetSerializeSize_impl(const prevector<N, T>& v, int nType, int nVersion, const V&) -{ - unsigned int nSize = GetSizeOfCompactSize(v.size()); - for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi) - nSize += GetSerializeSize((*vi), nType, nVersion); - return nSize; -} - -template<unsigned int N, typename T> -inline unsigned int GetSerializeSize(const prevector<N, T>& v, int nType, int nVersion) -{ - return GetSerializeSize_impl(v, nType, nVersion, T()); -} - - template<typename Stream, unsigned int N, typename T> -void Serialize_impl(Stream& os, const prevector<N, T>& v, int nType, int nVersion, const unsigned char&) +void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&) { WriteCompactSize(os, v.size()); if (!v.empty()) @@ -656,22 +603,22 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, int nType, int nVersio } template<typename Stream, unsigned int N, typename T, typename V> -void Serialize_impl(Stream& os, const prevector<N, T>& v, int nType, int nVersion, const V&) +void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&) { WriteCompactSize(os, v.size()); for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi) - ::Serialize(os, (*vi), nType, nVersion); + ::Serialize(os, (*vi)); } template<typename Stream, unsigned int N, typename T> -inline void Serialize(Stream& os, const prevector<N, T>& v, int nType, int nVersion) +inline void Serialize(Stream& os, const prevector<N, T>& v) { - Serialize_impl(os, v, nType, nVersion, T()); + Serialize_impl(os, v, T()); } template<typename Stream, unsigned int N, typename T> -void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, const unsigned char&) +void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&) { // Limit size per read so bogus size value won't cause out of memory v.clear(); @@ -687,7 +634,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, c } template<typename Stream, unsigned int N, typename T, typename V> -void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, const V&) +void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&) { v.clear(); unsigned int nSize = ReadCompactSize(is); @@ -700,14 +647,14 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, int nType, int nVersion, c nMid = nSize; v.resize(nMid); for (; i < nMid; i++) - Unserialize(is, v[i], nType, nVersion); + Unserialize(is, v[i]); } } template<typename Stream, unsigned int N, typename T> -inline void Unserialize(Stream& is, prevector<N, T>& v, int nType, int nVersion) +inline void Unserialize(Stream& is, prevector<N, T>& v) { - Unserialize_impl(is, v, nType, nVersion, T()); + Unserialize_impl(is, v, T()); } @@ -715,30 +662,8 @@ inline void Unserialize(Stream& is, prevector<N, T>& v, int nType, int nVersion) /** * vector */ -template<typename T, typename A> -unsigned int GetSerializeSize_impl(const std::vector<T, A>& v, int nType, int nVersion, const unsigned char&) -{ - return (GetSizeOfCompactSize(v.size()) + v.size() * sizeof(T)); -} - -template<typename T, typename A, typename V> -unsigned int GetSerializeSize_impl(const std::vector<T, A>& v, int nType, int nVersion, const V&) -{ - unsigned int nSize = GetSizeOfCompactSize(v.size()); - for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi) - nSize += GetSerializeSize((*vi), nType, nVersion); - return nSize; -} - -template<typename T, typename A> -inline unsigned int GetSerializeSize(const std::vector<T, A>& v, int nType, int nVersion) -{ - return GetSerializeSize_impl(v, nType, nVersion, T()); -} - - template<typename Stream, typename T, typename A> -void Serialize_impl(Stream& os, const std::vector<T, A>& v, int nType, int nVersion, const unsigned char&) +void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&) { WriteCompactSize(os, v.size()); if (!v.empty()) @@ -746,22 +671,22 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, int nType, int nVers } template<typename Stream, typename T, typename A, typename V> -void Serialize_impl(Stream& os, const std::vector<T, A>& v, int nType, int nVersion, const V&) +void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&) { WriteCompactSize(os, v.size()); for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi) - ::Serialize(os, (*vi), nType, nVersion); + ::Serialize(os, (*vi)); } template<typename Stream, typename T, typename A> -inline void Serialize(Stream& os, const std::vector<T, A>& v, int nType, int nVersion) +inline void Serialize(Stream& os, const std::vector<T, A>& v) { - Serialize_impl(os, v, nType, nVersion, T()); + Serialize_impl(os, v, T()); } template<typename Stream, typename T, typename A> -void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, const unsigned char&) +void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&) { // Limit size per read so bogus size value won't cause out of memory v.clear(); @@ -777,7 +702,7 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, } template<typename Stream, typename T, typename A, typename V> -void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, const V&) +void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&) { v.clear(); unsigned int nSize = ReadCompactSize(is); @@ -790,14 +715,14 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, int nType, int nVersion, nMid = nSize; v.resize(nMid); for (; i < nMid; i++) - Unserialize(is, v[i], nType, nVersion); + Unserialize(is, v[i]); } } template<typename Stream, typename T, typename A> -inline void Unserialize(Stream& is, std::vector<T, A>& v, int nType, int nVersion) +inline void Unserialize(Stream& is, std::vector<T, A>& v) { - Unserialize_impl(is, v, nType, nVersion, T()); + Unserialize_impl(is, v, T()); } @@ -805,24 +730,18 @@ inline void Unserialize(Stream& is, std::vector<T, A>& v, int nType, int nVersio /** * pair */ -template<typename K, typename T> -unsigned int GetSerializeSize(const std::pair<K, T>& item, int nType, int nVersion) -{ - return GetSerializeSize(item.first, nType, nVersion) + GetSerializeSize(item.second, nType, nVersion); -} - template<typename Stream, typename K, typename T> -void Serialize(Stream& os, const std::pair<K, T>& item, int nType, int nVersion) +void Serialize(Stream& os, const std::pair<K, T>& item) { - Serialize(os, item.first, nType, nVersion); - Serialize(os, item.second, nType, nVersion); + Serialize(os, item.first); + Serialize(os, item.second); } template<typename Stream, typename K, typename T> -void Unserialize(Stream& is, std::pair<K, T>& item, int nType, int nVersion) +void Unserialize(Stream& is, std::pair<K, T>& item) { - Unserialize(is, item.first, nType, nVersion); - Unserialize(is, item.second, nType, nVersion); + Unserialize(is, item.first); + Unserialize(is, item.second); } @@ -830,25 +749,16 @@ void Unserialize(Stream& is, std::pair<K, T>& item, int nType, int nVersion) /** * map */ -template<typename K, typename T, typename Pred, typename A> -unsigned int GetSerializeSize(const std::map<K, T, Pred, A>& m, int nType, int nVersion) -{ - unsigned int nSize = GetSizeOfCompactSize(m.size()); - for (typename std::map<K, T, Pred, A>::const_iterator mi = m.begin(); mi != m.end(); ++mi) - nSize += GetSerializeSize((*mi), nType, nVersion); - return nSize; -} - template<typename Stream, typename K, typename T, typename Pred, typename A> -void Serialize(Stream& os, const std::map<K, T, Pred, A>& m, int nType, int nVersion) +void Serialize(Stream& os, const std::map<K, T, Pred, A>& m) { WriteCompactSize(os, m.size()); for (typename std::map<K, T, Pred, A>::const_iterator mi = m.begin(); mi != m.end(); ++mi) - Serialize(os, (*mi), nType, nVersion); + Serialize(os, (*mi)); } template<typename Stream, typename K, typename T, typename Pred, typename A> -void Unserialize(Stream& is, std::map<K, T, Pred, A>& m, int nType, int nVersion) +void Unserialize(Stream& is, std::map<K, T, Pred, A>& m) { m.clear(); unsigned int nSize = ReadCompactSize(is); @@ -856,7 +766,7 @@ void Unserialize(Stream& is, std::map<K, T, Pred, A>& m, int nType, int nVersion for (unsigned int i = 0; i < nSize; i++) { std::pair<K, T> item; - Unserialize(is, item, nType, nVersion); + Unserialize(is, item); mi = m.insert(mi, item); } } @@ -866,25 +776,16 @@ void Unserialize(Stream& is, std::map<K, T, Pred, A>& m, int nType, int nVersion /** * set */ -template<typename K, typename Pred, typename A> -unsigned int GetSerializeSize(const std::set<K, Pred, A>& m, int nType, int nVersion) -{ - unsigned int nSize = GetSizeOfCompactSize(m.size()); - for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it) - nSize += GetSerializeSize((*it), nType, nVersion); - return nSize; -} - template<typename Stream, typename K, typename Pred, typename A> -void Serialize(Stream& os, const std::set<K, Pred, A>& m, int nType, int nVersion) +void Serialize(Stream& os, const std::set<K, Pred, A>& m) { WriteCompactSize(os, m.size()); for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it) - Serialize(os, (*it), nType, nVersion); + Serialize(os, (*it)); } template<typename Stream, typename K, typename Pred, typename A> -void Unserialize(Stream& is, std::set<K, Pred, A>& m, int nType, int nVersion) +void Unserialize(Stream& is, std::set<K, Pred, A>& m) { m.clear(); unsigned int nSize = ReadCompactSize(is); @@ -892,7 +793,7 @@ void Unserialize(Stream& is, std::set<K, Pred, A>& m, int nType, int nVersion) for (unsigned int i = 0; i < nSize; i++) { K key; - Unserialize(is, key, nType, nVersion); + Unserialize(is, key); it = m.insert(it, key); } } @@ -900,27 +801,61 @@ void Unserialize(Stream& is, std::set<K, Pred, A>& m, int nType, int nVersion) /** + * unique_ptr + */ +template<typename Stream, typename T> void +Serialize(Stream& os, const std::unique_ptr<const T>& p) +{ + Serialize(os, *p); +} + +template<typename Stream, typename T> +void Unserialize(Stream& is, std::unique_ptr<const T>& p) +{ + p.reset(new T(deserialize, is)); +} + + + +/** + * shared_ptr + */ +template<typename Stream, typename T> void +Serialize(Stream& os, const std::shared_ptr<const T>& p) +{ + Serialize(os, *p); +} + +template<typename Stream, typename T> +void Unserialize(Stream& is, std::shared_ptr<const T>& p) +{ + p = std::make_shared<const T>(deserialize, is); +} + + + +/** * Support for ADD_SERIALIZE_METHODS and READWRITE macro */ struct CSerActionSerialize { - bool ForRead() const { return false; } + constexpr bool ForRead() const { return false; } }; struct CSerActionUnserialize { - bool ForRead() const { return true; } + constexpr bool ForRead() const { return true; } }; template<typename Stream, typename T> -inline void SerReadWrite(Stream& s, const T& obj, int nType, int nVersion, CSerActionSerialize ser_action) +inline void SerReadWrite(Stream& s, const T& obj, CSerActionSerialize ser_action) { - ::Serialize(s, obj, nType, nVersion); + ::Serialize(s, obj); } template<typename Stream, typename T> -inline void SerReadWrite(Stream& s, T& obj, int nType, int nVersion, CSerActionUnserialize ser_action) +inline void SerReadWrite(Stream& s, T& obj, CSerActionUnserialize ser_action) { - ::Unserialize(s, obj, nType, nVersion); + ::Unserialize(s, obj); } @@ -931,33 +866,122 @@ inline void SerReadWrite(Stream& s, T& obj, int nType, int nVersion, CSerActionU +/* ::GetSerializeSize implementations + * + * Computing the serialized size of objects is done through a special stream + * object of type CSizeComputer, which only records the number of bytes written + * to it. + * + * If your Serialize or SerializationOp method has non-trivial overhead for + * serialization, it may be worthwhile to implement a specialized version for + * CSizeComputer, which uses the s.seek() method to record bytes that would + * be written instead. + */ class CSizeComputer { protected: size_t nSize; + const int nType; + const int nVersion; public: - int nType; - int nVersion; - CSizeComputer(int nTypeIn, int nVersionIn) : nSize(0), nType(nTypeIn), nVersion(nVersionIn) {} - CSizeComputer& write(const char *psz, size_t nSize) + void write(const char *psz, size_t _nSize) + { + this->nSize += _nSize; + } + + /** Pretend _nSize bytes are written, without specifying them. */ + void seek(size_t _nSize) { - this->nSize += nSize; - return *this; + this->nSize += _nSize; } template<typename T> CSizeComputer& operator<<(const T& obj) { - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } size_t size() const { return nSize; } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } }; +template<typename Stream> +void SerializeMany(Stream& s) +{ +} + +template<typename Stream, typename Arg> +void SerializeMany(Stream& s, Arg&& arg) +{ + ::Serialize(s, std::forward<Arg>(arg)); +} + +template<typename Stream, typename Arg, typename... Args> +void SerializeMany(Stream& s, Arg&& arg, Args&&... args) +{ + ::Serialize(s, std::forward<Arg>(arg)); + ::SerializeMany(s, std::forward<Args>(args)...); +} + +template<typename Stream> +inline void UnserializeMany(Stream& s) +{ +} + +template<typename Stream, typename Arg> +inline void UnserializeMany(Stream& s, Arg& arg) +{ + ::Unserialize(s, arg); +} + +template<typename Stream, typename Arg, typename... Args> +inline void UnserializeMany(Stream& s, Arg& arg, Args&... args) +{ + ::Unserialize(s, arg); + ::UnserializeMany(s, args...); +} + +template<typename Stream, typename... Args> +inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, Args&&... args) +{ + ::SerializeMany(s, std::forward<Args>(args)...); +} + +template<typename Stream, typename... Args> +inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&... args) +{ + ::UnserializeMany(s, args...); +} + +template<typename I> +inline void WriteVarInt(CSizeComputer &s, I n) +{ + s.seek(GetSizeOfVarInt<I>(n)); +} + +inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) +{ + s.seek(GetSizeOfCompactSize(nSize)); +} + +template <typename T> +size_t GetSerializeSize(const T& t, int nType, int nVersion = 0) +{ + return (CSizeComputer(nType, nVersion) << t).size(); +} + +template <typename S, typename T> +size_t GetSerializeSize(const S& s, const T& t) +{ + return (CSizeComputer(s.GetType(), s.GetVersion()) << t).size(); +} + #endif // BITCOIN_SERIALIZE_H diff --git a/src/streams.h b/src/streams.h index 7132364eb1..b508784238 100644 --- a/src/streams.h +++ b/src/streams.h @@ -26,17 +26,18 @@ template<typename Stream> class OverrideStream { Stream* stream; -public: + const int nType; const int nVersion; +public: OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} template<typename T> OverrideStream<Stream>& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this->stream, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } @@ -44,9 +45,22 @@ public: OverrideStream<Stream>& operator>>(T& obj) { // Unserialize from this stream - ::Unserialize(*this->stream, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } + + void write(const char* pch, size_t nSize) + { + stream->write(pch, nSize); + } + + void read(char* pch, size_t nSize) + { + stream->read(pch, nSize); + } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } }; template<typename S> @@ -55,6 +69,75 @@ OverrideStream<S> WithOrVersion(S* s, int nVersionFlag) return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag); } +/* Minimal stream for overwriting and/or appending to an existing byte vector + * + * The referenced vector will grow as necessary + */ +class CVectorWriter +{ + public: + +/* + * @param[in] nTypeIn Serialization Type + * @param[in] nVersionIn Serialization Version (including any flags) + * @param[in] vchDataIn Referenced byte vector to overwrite/append + * @param[in] nPosIn Starting position. Vector index where writes should start. The vector will initially + * grow as necessary to max(index, vec.size()). So to append, use vec.size(). +*/ + CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn) + { + if(nPos > vchData.size()) + vchData.resize(nPos); + } +/* + * (other params same as above) + * @param[in] args A list of items to serialize starting at nPos. +*/ + template <typename... Args> + CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn) + { + ::SerializeMany(*this, std::forward<Args>(args)...); + } + void write(const char* pch, size_t nSize) + { + assert(nPos <= vchData.size()); + size_t nOverwrite = std::min(nSize, vchData.size() - nPos); + if (nOverwrite) { + memcpy(vchData.data() + nPos, reinterpret_cast<const unsigned char*>(pch), nOverwrite); + } + if (nOverwrite < nSize) { + vchData.insert(vchData.end(), reinterpret_cast<const unsigned char*>(pch) + nOverwrite, reinterpret_cast<const unsigned char*>(pch) + nSize); + } + nPos += nSize; + } + template<typename T> + CVectorWriter& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } + int GetVersion() const + { + return nVersion; + } + int GetType() const + { + return nType; + } + void seek(size_t nSize) + { + nPos += nSize; + if(nPos > vchData.size()) + vchData.resize(nPos); + } +private: + const int nType; + const int nVersion; + std::vector<unsigned char>& vchData; + size_t nPos; +}; + /** Double ended buffer combining vector and stream-like interfaces. * * >> and << read and write unformatted data using the above serialization templates. @@ -66,9 +149,10 @@ protected: typedef CSerializeData vector_type; vector_type vch; unsigned int nReadPos; -public: + int nType; int nVersion; +public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; @@ -112,6 +196,13 @@ public: Init(nTypeIn, nVersionIn); } + template <typename... Args> + CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + { + Init(nTypeIn, nVersionIn); + ::SerializeMany(*this, std::forward<Args>(args)...); + } + void Init(int nTypeIn, int nVersionIn) { nReadPos = 0; @@ -244,13 +335,11 @@ public: int in_avail() { return size(); } void SetType(int n) { nType = n; } - int GetType() { return nType; } + int GetType() const { return nType; } void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - void WriteVersion() { *this << nVersion; } + int GetVersion() const { return nVersion; } - CDataStream& read(char* pch, size_t nSize) + void read(char* pch, size_t nSize) { // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; @@ -263,14 +352,13 @@ public: memcpy(pch, &vch[nReadPos], nSize); nReadPos = 0; vch.clear(); - return (*this); + return; } memcpy(pch, &vch[nReadPos], nSize); nReadPos = nReadPosNext; - return (*this); } - CDataStream& ignore(int nSize) + void ignore(int nSize) { // Ignore from the beginning of the buffer if (nSize < 0) { @@ -283,21 +371,19 @@ public: throw std::ios_base::failure("CDataStream::ignore(): end of data"); nReadPos = 0; vch.clear(); - return (*this); + return; } nReadPos = nReadPosNext; - return (*this); } - CDataStream& write(const char* pch, size_t nSize) + void write(const char* pch, size_t nSize) { // Write to the end of the buffer vch.insert(vch.end(), pch, pch + nSize); - return (*this); } template<typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { // Special case: stream << stream concatenates like stream += stream if (!vch.empty()) @@ -305,17 +391,10 @@ public: } template<typename T> - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); - } - - template<typename T> CDataStream& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } @@ -323,7 +402,7 @@ public: CDataStream& operator>>(T& obj) { // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } @@ -378,17 +457,15 @@ private: CAutoFile(const CAutoFile&); CAutoFile& operator=(const CAutoFile&); - int nType; - int nVersion; - + const int nType; + const int nVersion; + FILE* file; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) { file = filenew; - nType = nTypeIn; - nVersion = nVersionIn; } ~CAutoFile() @@ -423,23 +500,18 @@ public: // // Stream subset // - void SetType(int n) { nType = n; } - int GetType() { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - void WriteVersion() { *this << nVersion; } + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } - CAutoFile& read(char* pch, size_t nSize) + void read(char* pch, size_t nSize) { if (!file) throw std::ios_base::failure("CAutoFile::read: file handle is NULL"); if (fread(pch, 1, nSize, file) != nSize) throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed"); - return (*this); } - CAutoFile& ignore(size_t nSize) + void ignore(size_t nSize) { if (!file) throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL"); @@ -450,23 +522,14 @@ public: throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); nSize -= nNow; } - return (*this); } - CAutoFile& write(const char* pch, size_t nSize) + void write(const char* pch, size_t nSize) { if (!file) throw std::ios_base::failure("CAutoFile::write: file handle is NULL"); if (fwrite(pch, 1, nSize, file) != nSize) throw std::ios_base::failure("CAutoFile::write: write failed"); - return (*this); - } - - template<typename T> - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); } template<typename T> @@ -475,7 +538,7 @@ public: // Serialize to this stream if (!file) throw std::ios_base::failure("CAutoFile::operator<<: file handle is NULL"); - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } @@ -485,7 +548,7 @@ public: // Unserialize from this stream if (!file) throw std::ios_base::failure("CAutoFile::operator>>: file handle is NULL"); - ::Unserialize(*this, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } }; @@ -503,8 +566,8 @@ private: CBufferedFile(const CBufferedFile&); CBufferedFile& operator=(const CBufferedFile&); - int nType; - int nVersion; + const int nType; + const int nVersion; FILE *src; // source file uint64_t nSrcPos; // how many bytes have been read from source @@ -534,11 +597,9 @@ protected: public: CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : - nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0) + nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0) { src = fileIn; - nType = nTypeIn; - nVersion = nVersionIn; } ~CBufferedFile() @@ -546,6 +607,9 @@ public: fclose(); } + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } + void fclose() { if (src) { @@ -560,7 +624,7 @@ public: } // read a number of bytes - CBufferedFile& read(char *pch, size_t nSize) { + void read(char *pch, size_t nSize) { if (nSize + nReadPos > nReadLimit) throw std::ios_base::failure("Read attempted past buffer limit"); if (nSize + nRewind > vchBuf.size()) @@ -579,7 +643,6 @@ public: pch += nNow; nSize -= nNow; } - return (*this); } // return the current reading position @@ -625,7 +688,7 @@ public: template<typename T> CBufferedFile& operator>>(T& obj) { // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 1ec40fe830..67064314ef 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -6,7 +6,8 @@ #ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H -#include "support/pagelocker.h" +#include "support/lockedpool.h" +#include "support/cleanse.h" #include <string> @@ -39,20 +40,15 @@ struct secure_allocator : public std::allocator<T> { T* allocate(std::size_t n, const void* hint = 0) { - T* p; - p = std::allocator<T>::allocate(n, hint); - if (p != NULL) - LockedPageManager::Instance().LockRange(p, sizeof(T) * n); - return p; + return static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); } void deallocate(T* p, std::size_t n) { if (p != NULL) { memory_cleanse(p, sizeof(T) * n); - LockedPageManager::Instance().UnlockRange(p, sizeof(T) * n); } - std::allocator<T>::deallocate(p, n); + LockedPoolManager::Instance().free(p); } }; diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp new file mode 100644 index 0000000000..01273c9791 --- /dev/null +++ b/src/support/lockedpool.cpp @@ -0,0 +1,385 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "support/lockedpool.h" +#include "support/cleanse.h" + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#ifdef WIN32 +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0501 +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#else +#include <sys/mman.h> // for mmap +#include <sys/resource.h> // for getrlimit +#include <limits.h> // for PAGESIZE +#include <unistd.h> // for sysconf +#endif + +#include <algorithm> + +LockedPoolManager* LockedPoolManager::_instance = NULL; +std::once_flag LockedPoolManager::init_flag; + +/*******************************************************************************/ +// Utilities +// +/** Align up to power of 2 */ +static inline size_t align_up(size_t x, size_t align) +{ + return (x + align - 1) & ~(align - 1); +} + +/*******************************************************************************/ +// Implementation: Arena + +Arena::Arena(void *base_in, size_t size_in, size_t alignment_in): + base(static_cast<char*>(base_in)), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in) +{ + // Start with one free chunk that covers the entire arena + chunks_free.emplace(base, size_in); +} + +Arena::~Arena() +{ +} + +void* Arena::alloc(size_t size) +{ + // Round to next multiple of alignment + size = align_up(size, alignment); + + // Don't handle zero-sized chunks + if (size == 0) + return nullptr; + + // Pick a large enough free-chunk + auto it = std::find_if(chunks_free.begin(), chunks_free.end(), + [=](const std::map<char*, size_t>::value_type& chunk){ return chunk.second >= size; }); + if (it == chunks_free.end()) + return nullptr; + + // Create the used-chunk, taking its space from the end of the free-chunk + auto alloced = chunks_used.emplace(it->first + it->second - size, size).first; + if (!(it->second -= size)) + chunks_free.erase(it); + return reinterpret_cast<void*>(alloced->first); +} + +/* extend the Iterator if other begins at its end */ +template <class Iterator, class Pair> bool extend(Iterator it, const Pair& other) { + if (it->first + it->second == other.first) { + it->second += other.second; + return true; + } + return false; +} + +void Arena::free(void *ptr) +{ + // Freeing the NULL pointer is OK. + if (ptr == nullptr) { + return; + } + + // Remove chunk from used map + auto i = chunks_used.find(static_cast<char*>(ptr)); + if (i == chunks_used.end()) { + throw std::runtime_error("Arena: invalid or double free"); + } + auto freed = *i; + chunks_used.erase(i); + + // Add space to free map, coalescing contiguous chunks + auto next = chunks_free.upper_bound(freed.first); + auto prev = (next == chunks_free.begin()) ? chunks_free.end() : std::prev(next); + if (prev == chunks_free.end() || !extend(prev, freed)) + prev = chunks_free.emplace_hint(next, freed); + if (next != chunks_free.end() && extend(prev, *next)) + chunks_free.erase(next); +} + +Arena::Stats Arena::stats() const +{ + Arena::Stats r{ 0, 0, 0, chunks_used.size(), chunks_free.size() }; + for (const auto& chunk: chunks_used) + r.used += chunk.second; + for (const auto& chunk: chunks_free) + r.free += chunk.second; + r.total = r.used + r.free; + return r; +} + +#ifdef ARENA_DEBUG +void printchunk(char* base, size_t sz, bool used) { + std::cout << + "0x" << std::hex << std::setw(16) << std::setfill('0') << base << + " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz << + " 0x" << used << std::endl; +} +void Arena::walk() const +{ + for (const auto& chunk: chunks_used) + printchunk(chunk.first, chunk.second, true); + std::cout << std::endl; + for (const auto& chunk: chunks_free) + printchunk(chunk.first, chunk.second, false); + std::cout << std::endl; +} +#endif + +/*******************************************************************************/ +// Implementation: Win32LockedPageAllocator + +#ifdef WIN32 +/** LockedPageAllocator specialized for Windows. + */ +class Win32LockedPageAllocator: public LockedPageAllocator +{ +public: + Win32LockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +Win32LockedPageAllocator::Win32LockedPageAllocator() +{ + // Determine system page size in bytes + SYSTEM_INFO sSysInfo; + GetSystemInfo(&sSysInfo); + page_size = sSysInfo.dwPageSize; +} +void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + len = align_up(len, page_size); + void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (addr) { + // VirtualLock is used to attempt to keep keying material out of swap. Note + // that it does not provide this as a guarantee, but, in practice, memory + // that has been VirtualLock'd almost never gets written to the pagefile + // except in rare circumstances where memory is extremely low. + *lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0; + } + return addr; +} +void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + VirtualUnlock(const_cast<void*>(addr), len); +} + +size_t Win32LockedPageAllocator::GetLimit() +{ + // TODO is there a limit on windows, how to get it? + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: PosixLockedPageAllocator + +#ifndef WIN32 +/** LockedPageAllocator specialized for OSes that don't try to be + * special snowflakes. + */ +class PosixLockedPageAllocator: public LockedPageAllocator +{ +public: + PosixLockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +PosixLockedPageAllocator::PosixLockedPageAllocator() +{ + // Determine system page size in bytes +#if defined(PAGESIZE) // defined in limits.h + page_size = PAGESIZE; +#else // assume some POSIX OS + page_size = sysconf(_SC_PAGESIZE); +#endif +} + +// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define +// MAP_ANON which is deprecated +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + void *addr; + len = align_up(len, page_size); + addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (addr) { + *lockingSuccess = mlock(addr, len) == 0; + } + return addr; +} +void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + munlock(addr, len); + munmap(addr, len); +} +size_t PosixLockedPageAllocator::GetLimit() +{ +#ifdef RLIMIT_MEMLOCK + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) { + if (rlim.rlim_cur != RLIM_INFINITY) { + return rlim.rlim_cur; + } + } +#endif + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: LockedPool + +LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in): + allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0) +{ +} + +LockedPool::~LockedPool() +{ +} +void* LockedPool::alloc(size_t size) +{ + std::lock_guard<std::mutex> lock(mutex); + + // Don't handle impossible sizes + if (size == 0 || size > ARENA_SIZE) + return nullptr; + + // Try allocating from each current arena + for (auto &arena: arenas) { + void *addr = arena.alloc(size); + if (addr) { + return addr; + } + } + // If that fails, create a new one + if (new_arena(ARENA_SIZE, ARENA_ALIGN)) { + return arenas.back().alloc(size); + } + return nullptr; +} + +void LockedPool::free(void *ptr) +{ + std::lock_guard<std::mutex> lock(mutex); + // TODO we can do better than this linear search by keeping a map of arena + // extents to arena, and looking up the address. + for (auto &arena: arenas) { + if (arena.addressInArena(ptr)) { + arena.free(ptr); + return; + } + } + throw std::runtime_error("LockedPool: invalid address not pointing to any arena"); +} + +LockedPool::Stats LockedPool::stats() const +{ + std::lock_guard<std::mutex> lock(mutex); + LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0}; + for (const auto &arena: arenas) { + Arena::Stats i = arena.stats(); + r.used += i.used; + r.free += i.free; + r.total += i.total; + r.chunks_used += i.chunks_used; + r.chunks_free += i.chunks_free; + } + return r; +} + +bool LockedPool::new_arena(size_t size, size_t align) +{ + bool locked; + // If this is the first arena, handle this specially: Cap the upper size + // by the process limit. This makes sure that the first arena will at least + // be locked. An exception to this is if the process limit is 0: + // in this case no memory can be locked at all so we'll skip past this logic. + if (arenas.empty()) { + size_t limit = allocator->GetLimit(); + if (limit > 0) { + size = std::min(size, limit); + } + } + void *addr = allocator->AllocateLocked(size, &locked); + if (!addr) { + return false; + } + if (locked) { + cumulative_bytes_locked += size; + } else if (lf_cb) { // Call the locking-failed callback if locking failed + if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed. + allocator->FreeLocked(addr, size); + return false; + } + } + arenas.emplace_back(allocator.get(), addr, size, align); + return true; +} + +LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in): + Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in) +{ +} +LockedPool::LockedPageArena::~LockedPageArena() +{ + allocator->FreeLocked(base, size); +} + +/*******************************************************************************/ +// Implementation: LockedPoolManager +// +LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator): + LockedPool(std::move(allocator), &LockedPoolManager::LockingFailed) +{ +} + +bool LockedPoolManager::LockingFailed() +{ + // TODO: log something but how? without including util.h + return true; +} + +void LockedPoolManager::CreateInstance() +{ + // Using a local static instance guarantees that the object is initialized + // when it's first needed and also deinitialized after all objects that use + // it are done with it. I can think of one unlikely scenario where we may + // have a static deinitialization order/problem, but the check in + // LockedPoolManagerBase's destructor helps us detect if that ever happens. +#ifdef WIN32 + std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator()); +#else + std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator()); +#endif + static LockedPoolManager instance(std::move(allocator)); + LockedPoolManager::_instance = &instance; +} diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h new file mode 100644 index 0000000000..f5212bc266 --- /dev/null +++ b/src/support/lockedpool.h @@ -0,0 +1,231 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H +#define BITCOIN_SUPPORT_LOCKEDPOOL_H + +#include <stdint.h> +#include <list> +#include <map> +#include <mutex> +#include <memory> + +/** + * OS-dependent allocation and deallocation of locked/pinned memory pages. + * Abstract base class. + */ +class LockedPageAllocator +{ +public: + virtual ~LockedPageAllocator() {} + /** Allocate and lock memory pages. + * If len is not a multiple of the system page size, it is rounded up. + * Returns 0 in case of allocation failure. + * + * If locking the memory pages could not be accomplished it will still + * return the memory, however the lockingSuccess flag will be false. + * lockingSuccess is undefined if the allocation fails. + */ + virtual void* AllocateLocked(size_t len, bool *lockingSuccess) = 0; + + /** Unlock and free memory pages. + * Clear the memory before unlocking. + */ + virtual void FreeLocked(void* addr, size_t len) = 0; + + /** Get the total limit on the amount of memory that may be locked by this + * process, in bytes. Return size_t max if there is no limit or the limit + * is unknown. Return 0 if no memory can be locked at all. + */ + virtual size_t GetLimit() = 0; +}; + +/* An arena manages a contiguous region of memory by dividing it into + * chunks. + */ +class Arena +{ +public: + Arena(void *base, size_t size, size_t alignment); + virtual ~Arena(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t chunks_used; + size_t chunks_free; + }; + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get arena usage statistics */ + Stats stats() const; + +#ifdef ARENA_DEBUG + void walk() const; +#endif + + /** Return whether a pointer points inside this arena. + * This returns base <= ptr < (base+size) so only use it for (inclusive) + * chunk starting addresses. + */ + bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; } +private: + Arena(const Arena& other) = delete; // non construction-copyable + Arena& operator=(const Arena&) = delete; // non copyable + + /** Map of chunk address to chunk information. This class makes use of the + * sorted order to merge previous and next chunks during deallocation. + */ + std::map<char*, size_t> chunks_free; + std::map<char*, size_t> chunks_used; + /** Base address of arena */ + char* base; + /** End address of arena */ + char* end; + /** Minimum chunk alignment */ + size_t alignment; +}; + +/** Pool for locked memory chunks. + * + * To avoid sensitive key data from being swapped to disk, the memory in this pool + * is locked/pinned. + * + * An arena manages a contiguous region of memory. The pool starts out with one arena + * but can grow to multiple arenas if the need arises. + * + * Unlike a normal C heap, the administrative structures are separate from the managed + * memory. This has been done as the sizes and bases of objects are not in themselves sensitive + * information, as to conserve precious locked memory. In some operating systems + * the amount of memory that can be locked is small. + */ +class LockedPool +{ +public: + /** Size of one arena of locked memory. This is a compromise. + * Do not set this too low, as managing many arenas will increase + * allocation and deallocation overhead. Setting it too high allocates + * more locked memory from the OS than strictly necessary. + */ + static const size_t ARENA_SIZE = 256*1024; + /** Chunk alignment. Another compromise. Setting this too high will waste + * memory, setting it too low will facilitate fragmentation. + */ + static const size_t ARENA_ALIGN = 16; + + /** Callback when allocation succeeds but locking fails. + */ + typedef bool (*LockingFailed_Callback)(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t locked; + size_t chunks_used; + size_t chunks_free; + }; + + /** Create a new LockedPool. This takes ownership of the MemoryPageLocker, + * you can only instantiate this with LockedPool(std::move(...)). + * + * The second argument is an optional callback when locking a newly allocated arena failed. + * If this callback is provided and returns false, the allocation fails (hard fail), if + * it returns true the allocation proceeds, but it could warn. + */ + LockedPool(std::unique_ptr<LockedPageAllocator> allocator, LockingFailed_Callback lf_cb_in = 0); + ~LockedPool(); + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get pool usage statistics */ + Stats stats() const; +private: + LockedPool(const LockedPool& other) = delete; // non construction-copyable + LockedPool& operator=(const LockedPool&) = delete; // non copyable + + std::unique_ptr<LockedPageAllocator> allocator; + + /** Create an arena from locked pages */ + class LockedPageArena: public Arena + { + public: + LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align); + ~LockedPageArena(); + private: + void *base; + size_t size; + LockedPageAllocator *allocator; + }; + + bool new_arena(size_t size, size_t align); + + std::list<LockedPageArena> arenas; + LockingFailed_Callback lf_cb; + size_t cumulative_bytes_locked; + /** Mutex protects access to this pool's data structures, including arenas. + */ + mutable std::mutex mutex; +}; + +/** + * Singleton class to keep track of locked (ie, non-swappable) memory, for use in + * std::allocator templates. + * + * Some implementations of the STL allocate memory in some constructors (i.e., see + * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) + * Due to the unpredictable order of static initializers, we have to make sure the + * LockedPoolManager instance exists before any other STL-based objects that use + * secure_allocator are created. So instead of having LockedPoolManager also be + * static-initialized, it is created on demand. + */ +class LockedPoolManager : public LockedPool +{ +public: + /** Return the current instance, or create it once */ + static LockedPoolManager& Instance() + { + std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + return *LockedPoolManager::_instance; + } + +private: + LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator); + + /** Create a new LockedPoolManager specialized to the OS */ + static void CreateInstance(); + /** Called when locking fails, warn the user here */ + static bool LockingFailed(); + + static LockedPoolManager* _instance; + static std::once_flag init_flag; +}; + +#endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/support/pagelocker.cpp b/src/support/pagelocker.cpp deleted file mode 100644 index 7cea2d88c5..0000000000 --- a/src/support/pagelocker.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "support/pagelocker.h" - -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#define WIN32_LEAN_AND_MEAN 1 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include <windows.h> -// This is used to attempt to keep keying material out of swap -// Note that VirtualLock does not provide this as a guarantee on Windows, -// but, in practice, memory that has been VirtualLock'd almost never gets written to -// the pagefile except in rare circumstances where memory is extremely low. -#else -#include <sys/mman.h> -#include <limits.h> // for PAGESIZE -#include <unistd.h> // for sysconf -#endif - -LockedPageManager* LockedPageManager::_instance = NULL; -boost::once_flag LockedPageManager::init_flag = BOOST_ONCE_INIT; - -/** Determine system page size in bytes */ -static inline size_t GetSystemPageSize() -{ - size_t page_size; -#if defined(WIN32) - SYSTEM_INFO sSysInfo; - GetSystemInfo(&sSysInfo); - page_size = sSysInfo.dwPageSize; -#elif defined(PAGESIZE) // defined in limits.h - page_size = PAGESIZE; -#else // assume some POSIX OS - page_size = sysconf(_SC_PAGESIZE); -#endif - return page_size; -} - -bool MemoryPageLocker::Lock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualLock(const_cast<void*>(addr), len) != 0; -#else - return mlock(addr, len) == 0; -#endif -} - -bool MemoryPageLocker::Unlock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualUnlock(const_cast<void*>(addr), len) != 0; -#else - return munlock(addr, len) == 0; -#endif -} - -LockedPageManager::LockedPageManager() : LockedPageManagerBase<MemoryPageLocker>(GetSystemPageSize()) -{ -} diff --git a/src/support/pagelocker.h b/src/support/pagelocker.h deleted file mode 100644 index 538bf39453..0000000000 --- a/src/support/pagelocker.h +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SUPPORT_PAGELOCKER_H -#define BITCOIN_SUPPORT_PAGELOCKER_H - -#include "support/cleanse.h" - -#include <map> - -#include <boost/thread/mutex.hpp> -#include <boost/thread/once.hpp> - -/** - * Thread-safe class to keep track of locked (ie, non-swappable) memory pages. - * - * Memory locks do not stack, that is, pages which have been locked several times by calls to mlock() - * will be unlocked by a single call to munlock(). This can result in keying material ending up in swap when - * those functions are used naively. This class simulates stacking memory locks by keeping a counter per page. - * - * @note By using a map from each page base address to lock count, this class is optimized for - * small objects that span up to a few pages, mostly smaller than a page. To support large allocations, - * something like an interval tree would be the preferred data structure. - */ -template <class Locker> -class LockedPageManagerBase -{ -public: - LockedPageManagerBase(size_t _page_size) : page_size(_page_size) - { - // Determine bitmask for extracting page from address - assert(!(_page_size & (_page_size - 1))); // size must be power of two - page_mask = ~(_page_size - 1); - } - - ~LockedPageManagerBase() - { - } - - - // For all pages in affected range, increase lock count - void LockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - if (it == histogram.end()) // Newly locked page - { - locker.Lock(reinterpret_cast<void*>(page), page_size); - histogram.insert(std::make_pair(page, 1)); - } else // Page was already locked; increase counter - { - it->second += 1; - } - } - } - - // For all pages in affected range, decrease lock count - void UnlockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - assert(it != histogram.end()); // Cannot unlock an area that was not locked - // Decrease counter for page, when it is zero, the page will be unlocked - it->second -= 1; - if (it->second == 0) // Nothing on the page anymore that keeps it locked - { - // Unlock page and remove the count from histogram - locker.Unlock(reinterpret_cast<void*>(page), page_size); - histogram.erase(it); - } - } - } - - // Get number of locked pages for diagnostics - int GetLockedPageCount() - { - boost::mutex::scoped_lock lock(mutex); - return histogram.size(); - } - -private: - Locker locker; - boost::mutex mutex; - size_t page_size, page_mask; - // map of page base address to lock count - typedef std::map<size_t, int> Histogram; - Histogram histogram; -}; - - -/** - * OS-dependent memory page locking/unlocking. - * Defined as policy class to make stubbing for test possible. - */ -class MemoryPageLocker -{ -public: - /** Lock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Lock(const void* addr, size_t len); - /** Unlock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Unlock(const void* addr, size_t len); -}; - -/** - * Singleton class to keep track of locked (ie, non-swappable) memory pages, for use in - * std::allocator templates. - * - * Some implementations of the STL allocate memory in some constructors (i.e., see - * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) - * Due to the unpredictable order of static initializers, we have to make sure the - * LockedPageManager instance exists before any other STL-based objects that use - * secure_allocator are created. So instead of having LockedPageManager also be - * static-initialized, it is created on demand. - */ -class LockedPageManager : public LockedPageManagerBase<MemoryPageLocker> -{ -public: - static LockedPageManager& Instance() - { - boost::call_once(LockedPageManager::CreateInstance, LockedPageManager::init_flag); - return *LockedPageManager::_instance; - } - -private: - LockedPageManager(); - - static void CreateInstance() - { - // Using a local static instance guarantees that the object is initialized - // when it's first needed and also deinitialized after all objects that use - // it are done with it. I can think of one unlikely scenario where we may - // have a static deinitialization order/problem, but the check in - // LockedPageManagerBase's destructor helps us detect if that ever happens. - static LockedPageManager instance; - LockedPageManager::_instance = &instance; - } - - static LockedPageManager* _instance; - static boost::once_flag init_flag; -}; - -// -// Functions for directly locking/unlocking memory objects. -// Intended for non-dynamically allocated structures. -// -template <typename T> -void LockObject(const T& t) -{ - LockedPageManager::Instance().LockRange((void*)(&t), sizeof(T)); -} - -template <typename T> -void UnlockObject(const T& t) -{ - memory_cleanse((void*)(&t), sizeof(T)); - LockedPageManager::Instance().UnlockRange((void*)(&t), sizeof(T)); -} - -#endif // BITCOIN_SUPPORT_PAGELOCKER_H diff --git a/src/test/Checkpoints_tests.cpp b/src/test/Checkpoints_tests.cpp deleted file mode 100644 index 1b7d368e13..0000000000 --- a/src/test/Checkpoints_tests.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -// -// Unit tests for block-chain checkpoints -// - -#include "checkpoints.h" - -#include "uint256.h" -#include "test/test_bitcoin.h" -#include "chainparams.h" - -#include <boost/test/unit_test.hpp> - -using namespace std; - -BOOST_FIXTURE_TEST_SUITE(Checkpoints_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(sanity) -{ - const CCheckpointData& checkpoints = Params(CBaseChainParams::MAIN).Checkpoints(); - BOOST_CHECK(Checkpoints::GetTotalBlocksEstimate(checkpoints) >= 134444); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 97abeb7211..6eed636080 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -48,8 +48,9 @@ BOOST_AUTO_TEST_CASE(DoS_banning) { connman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, "", true); - GetNodeSignals().InitializeNode(dummyNode1.GetId(), &dummyNode1); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, "", true); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); // Should get banned SendMessages(&dummyNode1, *connman); @@ -57,8 +58,9 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, "", true); - GetNodeSignals().InitializeNode(dummyNode2.GetId(), &dummyNode2); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, "", true); + dummyNode2.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode2, *connman); dummyNode2.nVersion = 1; Misbehaving(dummyNode2.GetId(), 50); SendMessages(&dummyNode2, *connman); @@ -74,8 +76,9 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) connman->ClearBanned(); mapArgs["-banscore"] = "111"; // because 11 is my favorite number CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, "", true); - GetNodeSignals().InitializeNode(dummyNode1.GetId(), &dummyNode1); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, "", true); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); SendMessages(&dummyNode1, *connman); @@ -96,8 +99,9 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, "", true); - GetNodeSignals().InitializeNode(dummyNode.GetId(), &dummyNode); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, "", true); + dummyNode.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode, *connman); dummyNode.nVersion = 1; Misbehaving(dummyNode.GetId(), 100); diff --git a/src/test/README.md b/src/test/README.md index 3afdefe5fc..8f99804e10 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -1,4 +1,36 @@ -# Notes +### Compiling/running unit tests + +Unit tests will be automatically compiled if dependencies were met in `./configure` +and tests weren't explicitly disabled. + +After configuring, they can be run with `make check`. + +To run the bitcoind tests manually, launch `src/test/test_bitcoin`. + +To add more bitcoind tests, add `BOOST_AUTO_TEST_CASE` functions to the existing +.cpp files in the `test/` directory or add new .cpp files that +implement new BOOST_AUTO_TEST_SUITE sections. + +To run the bitcoin-qt tests manually, launch `src/qt/test/test_bitcoin-qt` + +To add more bitcoin-qt tests, add them to the `src/qt/test/` directory and +the `src/qt/test/test_main.cpp` file. + +### Running individual tests + +test_bitcoin has some built-in command-line arguments; for +example, to run just the getarg_tests verbosely: + + test_bitcoin --log_level=all --run_test=getarg_tests + +... or to run just the doubledash test: + + test_bitcoin --run_test=getarg_tests/doubledash + +Run `test_bitcoin --help` for the full list. + +### Note on adding test cases + The sources in this directory are unit test cases. Boost includes a unit testing framework, and since bitcoin already uses boost, it makes sense to simply use this framework rather than require developers to @@ -19,17 +51,6 @@ For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: [http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). -test_bitcoin has some built-in command-line arguments; for -example, to run just the getarg_tests verbosely: - - test_bitcoin --log_level=all --run_test=getarg_tests - -... or to run just the doubledash test: - - test_bitcoin --run_test=getarg_tests/doubledash - -Run `test_bitcoin --help` for the full list. - ### bitcoin-util-test.py The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just bitcoin-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory: @@ -37,4 +58,4 @@ The test directory also contains the bitcoin-util-test.py tool, which tests bitc ``` test/bitcoin-util-test.py --srcdir=[current directory] -```
\ No newline at end of file +``` diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 613f6c12d7..77e9df5d82 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -11,110 +11,224 @@ BOOST_FIXTURE_TEST_SUITE(allocator_tests, BasicTestingSetup) -// Dummy memory page locker for platform independent tests -static const void *last_lock_addr, *last_unlock_addr; -static size_t last_lock_len, last_unlock_len; -class TestLocker +BOOST_AUTO_TEST_CASE(arena_tests) { -public: - bool Lock(const void *addr, size_t len) + // Fake memory base address for testing + // without actually using memory. + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + void *chunk = b.alloc(1000); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(chunk != nullptr); + BOOST_CHECK(b.stats().used == 1008); // Aligned to 16 + BOOST_CHECK(b.stats().total == synth_size); // Nothing has disappeared? + b.free(chunk); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK(b.stats().free == synth_size); + try { // Test exception on double-free + b.free(chunk); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - last_lock_addr = addr; - last_lock_len = len; - return true; } - bool Unlock(const void *addr, size_t len) - { - last_unlock_addr = addr; - last_unlock_len = len; - return true; + + void *a0 = b.alloc(128); + void *a1 = b.alloc(256); + void *a2 = b.alloc(512); + BOOST_CHECK(b.stats().used == 896); + BOOST_CHECK(b.stats().total == synth_size); +#ifdef ARENA_DEBUG + b.walk(); +#endif + b.free(a0); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 768); + b.free(a1); + BOOST_CHECK(b.stats().used == 512); + void *a3 = b.alloc(128); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 640); + b.free(a2); + BOOST_CHECK(b.stats().used == 128); + b.free(a3); + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK_EQUAL(b.stats().chunks_used, 0); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + BOOST_CHECK_EQUAL(b.stats().chunks_free, 1); + + std::vector<void*> addr; + BOOST_CHECK(b.alloc(0) == nullptr); // allocating 0 always returns nullptr +#ifdef ARENA_DEBUG + b.walk(); +#endif + // Sweeping allocate all memory + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + BOOST_CHECK(b.stats().free == 0); + BOOST_CHECK(b.alloc(1024) == nullptr); // memory is full, this must return nullptr + BOOST_CHECK(b.alloc(0) == nullptr); + for (int x=0; x<1024; ++x) + b.free(addr[x]); + addr.clear(); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + + // Now in the other direction... + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + for (int x=0; x<1024; ++x) + b.free(addr[1023-x]); + addr.clear(); + + // Now allocate in smaller unequal chunks, then deallocate haphazardly + // Not all the chunks will succeed allocating, but freeing nullptr is + // allowed so that is no problem. + for (int x=0; x<2048; ++x) + addr.push_back(b.alloc(x+1)); + for (int x=0; x<2048; ++x) + b.free(addr[((x*23)%2048)^242]); + addr.clear(); + + // Go entirely wild: free and alloc interleaved, + // generate targets and sizes using pseudo-randomness. + for (int x=0; x<2048; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + for (int x=0; x<5000; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & 2047); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 } -}; + for (void *ptr: addr) + b.free(ptr); + addr.clear(); -BOOST_AUTO_TEST_CASE(test_LockedPageManagerBase) + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); +} + +/** Mock LockedPageAllocator for testing */ +class TestLockedPageAllocator: public LockedPageAllocator { - const size_t test_page_size = 4096; - LockedPageManagerBase<TestLocker> lpm(test_page_size); - size_t addr; - last_lock_addr = last_unlock_addr = 0; - last_lock_len = last_unlock_len = 0; - - /* Try large number of small objects */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; - } - /* Try small number of page-sized objects, straddling two pages */ - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* Try small number of page-sized objects aligned to exactly one page */ - addr = test_page_size*300; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* one very large object, straddling pages */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(600+500))); - /* one very large object, page aligned */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(1200+500-1))); - - BOOST_CHECK(lpm.GetLockedPageCount() == ( - (1000*33+test_page_size-1)/test_page_size + // small objects - 101 + 100 + // page-sized objects - 501 + 500)); // large objects - BOOST_CHECK((last_lock_len & (test_page_size-1)) == 0); // always lock entire pages - BOOST_CHECK(last_unlock_len == 0); // nothing unlocked yet - - /* And unlock again */ - addr = 0; - for(int i=0; i<1000; ++i) +public: + TestLockedPageAllocator(int count_in, int lockedcount_in): count(count_in), lockedcount(lockedcount_in) {} + void* AllocateLocked(size_t len, bool *lockingSuccess) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; + *lockingSuccess = false; + if (count > 0) { + --count; + + if (lockedcount > 0) { + --lockedcount; + *lockingSuccess = true; + } + + return reinterpret_cast<void*>(0x08000000 + (count<<24)); // Fake address, do not actually use this memory + } + return 0; } - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) + void FreeLocked(void* addr, size_t len) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; } - addr = test_page_size*300; - for(int i=0; i<100; ++i) + size_t GetLimit() { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; + return std::numeric_limits<size_t>::max(); } - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); +private: + int count; + int lockedcount; +}; - /* Check that everything is released */ - BOOST_CHECK(lpm.GetLockedPageCount() == 0); +BOOST_AUTO_TEST_CASE(lockedpool_tests_mock) +{ + // Test over three virtual arenas, of which one will succeed being locked + std::unique_ptr<LockedPageAllocator> x(new TestLockedPageAllocator(3, 1)); + LockedPool pool(std::move(x)); + BOOST_CHECK(pool.stats().total == 0); + BOOST_CHECK(pool.stats().locked == 0); - /* A few and unlocks of size zero (should have no effect) */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; - } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - addr = 0; - for(int i=0; i<1000; ++i) + // Ensure unreasonable requests are refused without allocating anything + void *invalid_toosmall = pool.alloc(0); + BOOST_CHECK(invalid_toosmall == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + void *invalid_toobig = pool.alloc(LockedPool::ARENA_SIZE+1); + BOOST_CHECK(invalid_toobig == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + + void *a0 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a0); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + void *a1 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a1); + void *a2 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a2); + void *a3 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a3); + void *a4 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a4); + void *a5 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a5); + // We've passed a count of three arenas, so this allocation should fail + void *a6 = pool.alloc(16); + BOOST_CHECK(!a6); + + pool.free(a0); + pool.free(a2); + pool.free(a4); + pool.free(a1); + pool.free(a3); + pool.free(a5); + BOOST_CHECK(pool.stats().total == 3*LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().used == 0); +} + +// These tests used the live LockedPoolManager object, this is also used +// by other tests so the conditions are somewhat less controllable and thus the +// tests are somewhat more error-prone. +BOOST_AUTO_TEST_CASE(lockedpool_tests_live) +{ + LockedPoolManager &pool = LockedPoolManager::Instance(); + LockedPool::Stats initial = pool.stats(); + + void *a0 = pool.alloc(16); + BOOST_CHECK(a0); + // Test reading and writing the allocated memory + *((uint32_t*)a0) = 0x1234; + BOOST_CHECK(*((uint32_t*)a0) == 0x1234); + + pool.free(a0); + try { // Test exception on double-free + pool.free(a0); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - BOOST_CHECK((last_unlock_len & (test_page_size-1)) == 0); // always unlock entire pages + // If more than one new arena was allocated for the above tests, something is wrong + BOOST_CHECK(pool.stats().total <= (initial.total + LockedPool::ARENA_SIZE)); + // Usage must be back to where it started + BOOST_CHECK(pool.stats().used == initial.used); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bctest.py b/src/test/bctest.py index d801415c70..adc5d0e418 100644 --- a/src/test/bctest.py +++ b/src/test/bctest.py @@ -1,4 +1,5 @@ -# Copyright 2014 BitPay, Inc. +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from __future__ import division,print_function,unicode_literals @@ -6,56 +7,115 @@ import subprocess import os import json import sys +import binascii +import difflib +import logging + +def parse_output(a, fmt): + """Parse the output according to specified format. + + Raise an error if the output can't be parsed.""" + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return binascii.a2b_hex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) def bctest(testDir, testObj, exeext): + """Runs a single test, comparing output and RC to expected output and RC. + + Raises an error if input can't be read, executable fails, or output/RC + are not as expected. Error is caught by bctester() and reported. + """ + # Get the exec names and arguments + execprog = testObj['exec'] + exeext + execargs = testObj['args'] + execrun = [execprog] + execargs + + # Read the input data (if there is any) + stdinCfg = None + inputData = None + if "input" in testObj: + filename = testDir + "/" + testObj['input'] + inputData = open(filename).read() + stdinCfg = subprocess.PIPE + + # Read the expected output data (if there is any) + outputFn = None + outputData = None + if "output_cmp" in testObj: + outputFn = testObj['output_cmp'] + outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) + try: + outputData = open(testDir + "/" + outputFn).read() + except: + logging.error("Output file " + outputFn + " can not be opened") + raise + if not outputData: + logging.error("Output data missing for " + outputFn) + raise Exception + + # Run the test + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) + try: + outs = proc.communicate(input=inputData) + except OSError: + logging.error("OSError, Failed to execute " + execprog) + raise + + if outputData: + # Parse command output and expected output + try: + a_parsed = parse_output(outs[0], outputType) + except Exception as e: + logging.error('Error parsing command output as %s: %s' % (outputType,e)) + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) + raise + # Compare data + if a_parsed != b_parsed: + logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + raise Exception + # Compare formatting + if outs[0] != outputData: + error_message = "Output formatting mismatch for " + outputFn + ":\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + outs[0].splitlines(True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + raise Exception + + # Compare the return code to the expected return code + wantRC = 0 + if "return_code" in testObj: + wantRC = testObj['return_code'] + if proc.returncode != wantRC: + logging.error("Return code mismatch for " + outputFn) + raise Exception + +def bctester(testDir, input_basename, buildenv): + """ Loads and parses the input file, runs all tests and reports results""" + input_filename = testDir + "/" + input_basename + raw_data = open(input_filename).read() + input_data = json.loads(raw_data) + + failed_testcases = [] - execprog = testObj['exec'] + exeext - execargs = testObj['args'] - execrun = [execprog] + execargs - stdinCfg = None - inputData = None - if "input" in testObj: - filename = testDir + "/" + testObj['input'] - inputData = open(filename).read() - stdinCfg = subprocess.PIPE - - outputFn = None - outputData = None - if "output_cmp" in testObj: - outputFn = testObj['output_cmp'] - outputData = open(testDir + "/" + outputFn).read() - if not outputData: - print("Output data missing for " + outputFn) - sys.exit(1) - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) - try: - outs = proc.communicate(input=inputData) - except OSError: - print("OSError, Failed to execute " + execprog) - sys.exit(1) - - if outputData and (outs[0] != outputData): - print("Output data mismatch for " + outputFn) - sys.exit(1) - - wantRC = 0 - if "return_code" in testObj: - wantRC = testObj['return_code'] - if proc.returncode != wantRC: - print("Return code mismatch for " + outputFn) - sys.exit(1) - -def bctester(testDir, input_basename, buildenv, verbose = False): - input_filename = testDir + "/" + input_basename - raw_data = open(input_filename).read() - input_data = json.loads(raw_data) - - for testObj in input_data: - if verbose and "description" in testObj: - print ("Testing: " + testObj["description"]) - bctest(testDir, testObj, buildenv.exeext) - if verbose and "description" in testObj: - print ("PASS") - - sys.exit(0) + for testObj in input_data: + try: + bctest(testDir, testObj, buildenv.exeext) + logging.info("PASSED: " + testObj["description"]) + except: + logging.info("FAILED: " + testObj["description"]) + failed_testcases.append(testObj["description"]) + if failed_testcases: + logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") + sys.exit(1) + else: + sys.exit(0) diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py index 3099506d6d..1c090b3f3f 100755 --- a/src/test/bitcoin-util-test.py +++ b/src/test/bitcoin-util-test.py @@ -1,27 +1,32 @@ #!/usr/bin/env python -# Copyright 2014 BitPay, Inc. +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from __future__ import division,print_function,unicode_literals import os +import sys import bctest import buildenv import argparse +import logging help_text="""Test framework for bitcoin utils. Runs automatically during `make check`. -Can also be run manually from the src directory by specifiying the source directory: +Can also be run manually from the src directory by specifying the source directory: -test/bitcoin-util-test.py --src=[srcdir] +test/bitcoin-util-test.py --srcdir='srcdir' [--verbose] """ - if __name__ == '__main__': - verbose = False + # Try to get the source directory from the environment variables. This will + # be set for `make check` automated runs. If environment variable is not set, + # then get the source directory from command line args. try: srcdir = os.environ["srcdir"] + verbose = False except: parser = argparse.ArgumentParser(description=help_text) parser.add_argument('-s', '--srcdir') @@ -29,4 +34,13 @@ if __name__ == '__main__': args = parser.parse_args() srcdir = args.srcdir verbose = args.verbose - bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose) + + if verbose: + level = logging.DEBUG + else: + level = logging.ERROR + formatter = '%(asctime)s - %(levelname)s - %(message)s' + # Add the format/level to the logger + logging.basicConfig(format = formatter, level=level) + + bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 7530b013bd..b013cda6d7 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -26,21 +26,21 @@ static CBlock BuildBlockTestCase() { tx.vout[0].nValue = 42; block.vtx.resize(3); - block.vtx[0] = tx; + block.vtx[0] = MakeTransactionRef(tx); block.nVersion = 42; block.hashPrevBlock = GetRandHash(); block.nBits = 0x207fffff; tx.vin[0].prevout.hash = GetRandHash(); tx.vin[0].prevout.n = 0; - block.vtx[1] = tx; + block.vtx[1] = MakeTransactionRef(tx); tx.vin.resize(10); for (size_t i = 0; i < tx.vin.size(); i++) { tx.vin[i].prevout.hash = GetRandHash(); tx.vin[i].prevout.n = 0; } - block.vtx[2] = tx; + block.vtx[2] = MakeTransactionRef(tx); bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); @@ -59,8 +59,8 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { @@ -78,14 +78,14 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::list<CTransaction> removed; - pool.removeRecursive(block.vtx[2], removed); + std::vector<CTransactionRef> removed; + pool.removeRecursive(*block.vtx[2], &removed); BOOST_CHECK_EQUAL(removed.size(), 1); CBlock block2; - std::vector<CTransaction> vtx_missing; + std::vector<CTransactionRef> vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions vtx_missing.push_back(block.vtx[2]); // Wrong transaction @@ -129,7 +129,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(header); READWRITE(nonce); size_t shorttxids_size = shorttxids.size(); @@ -152,8 +152,10 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[2].GetHash(), entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; // Test with pre-forwarding tx 1, but not coinbase { @@ -161,8 +163,8 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0].GetHash()); - shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2].GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); + shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; @@ -176,10 +178,10 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector<CTransaction> vtx_missing; + std::vector<CTransactionRef> vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_INVALID); // No transactions vtx_missing.push_back(block.vtx[1]); // Wrong transaction @@ -194,9 +196,13 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + txhash = block.vtx[2]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + block3.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -205,8 +211,10 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); - pool.addUnchecked(block.vtx[1].GetHash(), entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + pool.addUnchecked(block.vtx[1]->GetHash(), entry.FromTx(*block.vtx[1])); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + + uint256 txhash; // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool { @@ -215,7 +223,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; // id == 1 as it is 1 after index 1 shortIDs.shorttxids.resize(1); - shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1].GetHash()); + shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; @@ -229,19 +237,22 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; - std::vector<CTransaction> vtx_missing; + std::vector<CTransactionRef> vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + txhash = block.vtx[1]->GetHash(); + block.vtx.clear(); + block2.vtx.clear(); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) @@ -255,7 +266,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) CBlock block; block.vtx.resize(1); - block.vtx[0] = coinbase; + block.vtx[0] = MakeTransactionRef(std::move(coinbase)); block.nVersion = 42; block.hashPrevBlock = GetRandHash(); block.nBits = 0x207fffff; @@ -280,7 +291,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) BOOST_CHECK(partialBlock.IsTxAvailable(0)); CBlock block2; - std::vector<CTransaction> vtx_missing; + std::vector<CTransactionRef> vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 042fad42da..25fb9ea2b7 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -41,7 +41,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "BloomFilter doesn't contain just-inserted object (3)!"); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - filter.Serialize(stream, SER_NETWORK, PROTOCOL_VERSION); + stream << filter; vector<unsigned char> vch = ParseHex("03614e9b050000000000000001"); vector<char> expected(vch.size()); @@ -73,7 +73,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) BOOST_CHECK_MESSAGE(filter.contains(ParseHex("b9300670b4c5366e95b2699e8b18bc75e5f729c5")), "BloomFilter doesn't contain just-inserted object (3)!"); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - filter.Serialize(stream, SER_NETWORK, PROTOCOL_VERSION); + stream << filter; vector<unsigned char> vch = ParseHex("03ce4299050000000100008001"); vector<char> expected(vch.size()); @@ -100,7 +100,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) filter.insert(vector<unsigned char>(hash.begin(), hash.end())); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - filter.Serialize(stream, SER_NETWORK, PROTOCOL_VERSION); + stream << filter; vector<unsigned char> vch = ParseHex("038fc16b080000000000000001"); vector<char> expected(vch.size()); diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index b487686136..82de302053 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -3,11 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "coins.h" -#include "test_random.h" #include "script/standard.h" #include "uint256.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include "main.h" #include "consensus/validation.h" diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index c7b4fb240c..7dcd548edf 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -9,9 +9,9 @@ #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" -#include "test_random.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include <vector> diff --git a/src/test/data/bitcoin-util-test.json b/src/test/data/bitcoin-util-test.json index de95044597..98be75919b 100644 --- a/src/test/data/bitcoin-util-test.json +++ b/src/test/data/bitcoin-util-test.json @@ -103,6 +103,16 @@ "description": "Creates a new transaction with a single empty output script (output in json)" }, { "exec": "./bitcoin-tx", + "args": ["01000000000100000000000000000000000000"], + "output_cmp": "txcreate2.hex", + "description": "Parses a transation with no inputs and a single output script" + }, + { "exec": "./bitcoin-tx", + "args": ["-json", "01000000000100000000000000000000000000"], + "output_cmp": "txcreate2.json", + "description": "Parses a transation with no inputs and a single output script (output in json)" + }, + { "exec": "./bitcoin-tx", "args": ["-create", "in=4d49a71ec9da436f71ec4ee231d04f292a29cd316f598bb7068feccabdc59485:0", diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index 2f299aa5fe..27acb2f16a 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -53,7 +53,7 @@ ["The following tests for the presence of a bug in the handling of SIGHASH_SINGLE"], ["It results in signing the constant 1, instead of something generated based on the transaction,"], ["when the input doing the signing has an index greater than the maximum output index"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0xe52b482f2faa8ecbf0db344f93c84ac908557f33 EQUALVERIFY CHECKSIG"], ["0000000000000000000000000000000000000000000000000000000000000200", 0, "1"]], +[[["0000000000000000000000000000000000000000000000000000000000000200", 0, "1"], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0xe52b482f2faa8ecbf0db344f93c84ac908557f33 EQUALVERIFY CHECKSIG"]], "01000000020002000000000000000000000000000000000000000000000000000000000000000000000151ffffffff0001000000000000000000000000000000000000000000000000000000000000000000006b483045022100c9cdd08798a28af9d1baf44a6c77bcc7e279f47dc487c8c899911bc48feaffcc0220503c5c50ae3998a733263c5c0f7061b483e2b56c4c41b456e7d2f5a78a74c077032102d5c25adb51b61339d2b05315791e21bbe80ea470a49db0135720983c905aace0ffffffff010000000000000000015100000000", "P2SH"], ["An invalid P2SH Transaction"], @@ -334,9 +334,9 @@ "0100000000010100010000000000000000000000000000000000000000000000000000000000000000000023220020ff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100aa5d8aa40a90f23ce2c3d11bc845ca4a12acd99cbea37de6b9f6d86edebba8cb022022dedc2aa0a255f74d04c0b76ece2d7c691f9dd11a64a8ac49f62a99c3a05f9d01232103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ac00000000", "P2SH,WITNESS"], ["Witness with SigHash Single|AnyoneCanPay"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], +[[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], -["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], +["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], "0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff05540b0000000000000151d0070000000000000151840300000000000001513c0f00000000000001512c010000000000000151000248304502210092f4777a0f17bf5aeb8ae768dec5f2c14feabf9d1fe2c89c78dfed0f13fdb86902206da90a86042e252bcd1e80a168c719e4a1ddcc3cebea24b9812c5453c79107e9832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71000000000000", "P2SH,WITNESS"], @@ -359,9 +359,9 @@ "0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b000000000000015100024730440220699e6b0cfe015b64ca3283e6551440a34f901ba62dd4c72fe1cb815afb2e6761022021cc5e84db498b1479de14efda49093219441adc6c543e5534979605e273d80b032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], ["Witness with SigHash None|AnyoneCanPay"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], +[[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], +["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], -["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], "0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff04b60300000000000001519e070000000000000151860b00000000000001009600000000000000015100000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], @@ -390,9 +390,9 @@ "01000000000103000100000000000000000000000000000000000000000000000000000000000000000000000200000000010000000000000000000000000000000000000000000000000000000000000100000000ffffffff000100000000000000000000000000000000000000000000000000000000000002000000000200000003e8030000000000000151d0070000000000000151b80b00000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], ["Witness with SigHash All|AnyoneCanPay"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], +[[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], +["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], -["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], "0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000002483045022100a3cec69b52cba2d2de623eeef89e0ba1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], @@ -458,8 +458,8 @@ "0100000000010200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff02e8030000000000000151e90300000000000001510247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], ["Witness Single|AnyoneCanPay does not hash input's position (permutation)"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000], -["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1001]], +[[["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1001], +["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000]], "0100000000010200010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff02e9030000000000000151e80300000000000001510248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], ["Non witness Single|AnyoneCanPay hash input's position"], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index d4d825d199..2d791ee18d 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -254,7 +254,7 @@ struct StringContentsSerializer { ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { if (ser_action.ForRead()) { str.clear(); char c = 0; diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index faaddffad8..e3531486aa 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(limitedmap_test) // make sure the item is present BOOST_CHECK(map.count(i) == 1); - // use the iterator to check for the expected key adn value + // use the iterator to check for the expected key and value BOOST_CHECK(it->first == i); BOOST_CHECK(it->second == i + 1); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 033a50f94f..1faf8b6aeb 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -55,15 +55,15 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool testPool(CFeeRate(0)); - std::list<CTransaction> removed; + std::vector<CTransactionRef> removed; // Nothing in pool, remove should do nothing: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 1); removed.clear(); @@ -75,16 +75,16 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 2); removed.clear(); // ... make sure grandchild and child are gone: - testPool.removeRecursive(txGrandChild[0], removed); + testPool.removeRecursive(txGrandChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Remove parent, all children/grandchildren should go: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 5); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 6); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -281,12 +281,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10); // Now try removing tx10 and verify the sort order returns to normal - std::list<CTransaction> removed; - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx()); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), removed); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx()); + pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx()); /* Now check the sort on the mining score index. * Final order should be: * @@ -411,10 +410,9 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) CheckSort<ancestor_score>(pool, sortedOrder); /* after tx6 is mined, tx7 should move up in the sort */ - std::vector<CTransaction> vtx; - vtx.push_back(tx6); - std::list<CTransaction> dummy; - pool.removeForBlock(vtx, 1, dummy, false); + std::vector<CTransactionRef> vtx; + vtx.push_back(MakeTransactionRef(tx6)); + pool.removeForBlock(vtx, 1, NULL, false); sortedOrder.erase(sortedOrder.begin()+1); sortedOrder.pop_back(); @@ -548,13 +546,12 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(tx5.GetHash(), entry.Fee(1000LL).FromTx(tx5, &pool)); pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); - std::vector<CTransaction> vtx; - std::list<CTransaction> conflicts; + std::vector<CTransactionRef> vtx; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); // ... we should keep the same min fee until we get a block - pool.removeForBlock(vtx, 1, conflicts); + pool.removeForBlock(vtx, 1); SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/2); // ... then feerate should drop 1/2 each halflife diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 66ca381ea7..55e6852a15 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -4,7 +4,7 @@ #include "consensus/merkle.h" #include "test/test_bitcoin.h" -#include "test_random.h" +#include "test/test_random.h" #include <boost/test/unit_test.hpp> @@ -15,8 +15,8 @@ static uint256 BlockBuildMerkleTree(const CBlock& block, bool* fMutated, std::ve { vMerkleTree.clear(); vMerkleTree.reserve(block.vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. - for (std::vector<CTransaction>::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it) - vMerkleTree.push_back(it->GetHash()); + for (std::vector<CTransactionRef>::const_iterator it(block.vtx.begin()); it != block.vtx.end(); ++it) + vMerkleTree.push_back((*it)->GetHash()); int j = 0; bool mutated = false; for (int nSize = block.vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(merkle_test) for (int j = 0; j < ntx; j++) { CMutableTransaction mtx; mtx.nLockTime = j; - block.vtx[j] = mtx; + block.vtx[j] = MakeTransactionRef(std::move(mtx)); } // Compute the root of the block before mutating it. bool unmutatedMutated = false; @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(merkle_test) std::vector<uint256> newBranch = BlockMerkleBranch(block, mtx); std::vector<uint256> oldBranch = BlockGetMerkleBranch(block, merkleTree, mtx); BOOST_CHECK(oldBranch == newBranch); - BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx].GetHash(), newBranch, mtx) == oldRoot); + BOOST_CHECK(ComputeMerkleRootFromBranch(block.vtx[mtx]->GetHash(), newBranch, mtx) == oldRoot); } } } diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d6d7b5716e..85a2e907c2 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -77,7 +77,7 @@ bool TestSequenceLocks(const CTransaction &tx, int flags) // Implemented as an additional function, rather than a separate test case, // to allow reusing the blockchain created in CreateNewBlock_validity. // Note that this test assumes blockprioritysize is 0. -void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, std::vector<CTransaction *>& txFirst) +void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, std::vector<CTransactionRef>& txFirst) { // Test the ancestor feerate transaction selection. TestMemPoolEntryHelper entry; @@ -108,9 +108,9 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, mempool.addUnchecked(hashHighFeeTx, entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate->block.vtx[1].GetHash() == hashParentTx); - BOOST_CHECK(pblocktemplate->block.vtx[2].GetHash() == hashHighFeeTx); - BOOST_CHECK(pblocktemplate->block.vtx[3].GetHash() == hashMediumFeeTx); + BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); + BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); + BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashMediumFeeTx); // Test that a package below the min relay fee doesn't get included tx.vin[0].prevout.hash = hashHighFeeTx; @@ -130,21 +130,20 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { - BOOST_CHECK(pblocktemplate->block.vtx[i].GetHash() != hashFreeTx); - BOOST_CHECK(pblocktemplate->block.vtx[i].GetHash() != hashLowFeeTx); + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx); + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashLowFeeTx); } // Test that packages above the min relay fee do get included, even if one // of the transactions is below the min relay fee // Remove the low fee transaction and replace with a higher fee transaction - std::list<CTransaction> dummy; - mempool.removeRecursive(tx, dummy); + mempool.removeRecursive(tx); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx, entry.Fee(feeToUse+2).FromTx(tx)); pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate->block.vtx[4].GetHash() == hashFreeTx); - BOOST_CHECK(pblocktemplate->block.vtx[5].GetHash() == hashLowFeeTx); + BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); + BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); // Test that transaction selection properly updates ancestor fee // calculations as ancestor transactions get included in a block. @@ -167,8 +166,8 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, // Verify that this tx isn't selected. for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { - BOOST_CHECK(pblocktemplate->block.vtx[i].GetHash() != hashFreeTx2); - BOOST_CHECK(pblocktemplate->block.vtx[i].GetHash() != hashLowFeeTx2); + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx2); + BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashLowFeeTx2); } // This tx will be mineable, and should cause hashLowFeeTx2 to be selected @@ -177,7 +176,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee mempool.addUnchecked(tx.GetHash(), entry.Fee(10000).FromTx(tx)); pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate->block.vtx[8].GetHash() == hashLowFeeTx2); + BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } // NOTE: These tests rely on CreateNewBlock doing its own self-validation! @@ -204,28 +203,27 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // We can't make transactions until we have inputs // Therefore, load 100 blocks :) int baseheight = 0; - std::vector<CTransaction*>txFirst; + std::vector<CTransactionRef> txFirst; for (unsigned int i = 0; i < sizeof(blockinfo)/sizeof(*blockinfo); ++i) { CBlock *pblock = &pblocktemplate->block; // pointer for convenience pblock->nVersion = 1; pblock->nTime = chainActive.Tip()->GetMedianTimePast()+1; - CMutableTransaction txCoinbase(pblock->vtx[0]); + CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.nVersion = 1; txCoinbase.vin[0].scriptSig = CScript(); txCoinbase.vin[0].scriptSig.push_back(blockinfo[i].extranonce); txCoinbase.vin[0].scriptSig.push_back(chainActive.Height()); + txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); - pblock->vtx[0] = CTransaction(txCoinbase); + pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); if (txFirst.size() == 0) baseheight = chainActive.Height(); if (txFirst.size() < 4) - txFirst.push_back(new CTransaction(pblock->vtx[0])); + txFirst.push_back(pblock->vtx[0]); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); pblock->nNonce = blockinfo[i].nonce; - CValidationState state; - BOOST_CHECK(ProcessNewBlock(state, chainparams, NULL, pblock, true, NULL)); - BOOST_CHECK(state.IsValid()); + BOOST_CHECK(ProcessNewBlock(chainparams, pblock, true, NULL, NULL)); pblock->hashPrevBlock = pblock->GetHash(); } @@ -488,9 +486,6 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) TestPackageSelection(chainparams, scriptPubKey, txFirst); - BOOST_FOREACH(CTransaction *_tx, txFirst) - delete _tx; - fCheckpointsEnabled = true; } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index f4b5768693..87cb38daac 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -17,7 +17,7 @@ using namespace std; class CAddrManSerializationMock : public CAddrMan { public: - virtual void Serialize(CDataStream& s, int nType, int nVersionDummy) const = 0; + virtual void Serialize(CDataStream& s) const = 0; //! Ensure that bucket placement is always the same for testing purposes. void MakeDeterministic() @@ -30,16 +30,16 @@ public: class CAddrManUncorrupted : public CAddrManSerializationMock { public: - void Serialize(CDataStream& s, int nType, int nVersionDummy) const + void Serialize(CDataStream& s) const { - CAddrMan::Serialize(s, nType, nVersionDummy); + CAddrMan::Serialize(s); } }; class CAddrManCorrupted : public CAddrManSerializationMock { public: - void Serialize(CDataStream& s, int nType, int nVersionDummy) const + void Serialize(CDataStream& s) const { // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. unsigned char nVersion = 1; @@ -164,12 +164,12 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) bool fInboundIn = false; // Test that fFeeler is false by default. - CNode* pnode1 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, pszDest, fInboundIn); + CNode* pnode1 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, pszDest, fInboundIn); BOOST_CHECK(pnode1->fInbound == false); BOOST_CHECK(pnode1->fFeeler == false); fInboundIn = true; - CNode* pnode2 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, pszDest, fInboundIn); + CNode* pnode2 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, pszDest, fInboundIn); BOOST_CHECK(pnode2->fInbound == true); BOOST_CHECK(pnode2->fFeeler == false); } diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index b7f83d38f0..e6b689bc6c 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -9,8 +9,8 @@ #include "uint256.h" #include "arith_uint256.h" #include "version.h" -#include "test_random.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include <vector> @@ -45,14 +45,14 @@ BOOST_AUTO_TEST_CASE(pmt_test1) for (unsigned int j=0; j<nTx; j++) { CMutableTransaction tx; tx.nLockTime = j; // actual transaction data doesn't matter; just make the nLockTime's unique - block.vtx.push_back(CTransaction(tx)); + block.vtx.push_back(MakeTransactionRef(std::move(tx))); } // calculate actual merkle root and height uint256 merkleRoot1 = BlockMerkleRoot(block); std::vector<uint256> vTxid(nTx, uint256()); for (unsigned int j=0; j<nTx; j++) - vTxid[j] = block.vtx[j].GetHash(); + vTxid[j] = block.vtx[j]->GetHash(); int nHeight = 1, nTx_ = nTx; while (nTx_ > 1) { nTx_ = (nTx_+1)/2; diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 5c902387f1..7dc8f226c9 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -19,26 +19,18 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) CTxMemPool mpool(CFeeRate(1000)); TestMemPoolEntryHelper entry; CAmount basefee(2000); - double basepri = 1e6; CAmount deltaFee(100); - double deltaPri=5e5; - std::vector<CAmount> feeV[2]; - std::vector<double> priV[2]; + std::vector<CAmount> feeV; - // Populate vectors of increasing fees or priorities + // Populate vectors of increasing fees for (int j = 0; j < 10; j++) { - //V[0] is for fee transactions - feeV[0].push_back(basefee * (j+1)); - priV[0].push_back(0); - //V[1] is for priority transactions - feeV[1].push_back(CAmount(0)); - priV[1].push_back(basepri * pow(10, j+1)); + feeV.push_back(basefee * (j+1)); } // Store the hashes of transactions that have been - // added to the mempool by their associate fee/pri + // added to the mempool by their associate fee // txHashes[j] is populated with transactions either of - // fee = basefee * (j+1) OR pri = 10^6 * 10^(j+1) + // fee = basefee * (j+1) std::vector<uint256> txHashes[10]; // Create a transaction template @@ -46,7 +38,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (unsigned int i = 0; i < 128; i++) garbage.push_back('X'); CMutableTransaction tx; - std::list<CTransaction> dummyConflicted; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); @@ -54,34 +45,34 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) CFeeRate baseRate(basefee, GetVirtualTransactionSize(tx)); // Create a fake block - std::vector<CTransaction> block; + std::vector<CTransactionRef> block; int blocknum = 0; // Loop through 200 blocks // At a decay .998 and 4 fee transactions per block // This makes the tx count about 1.33 per bucket, above the 1 threshold while (blocknum < 200) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); txHashes[j].push_back(hash); } } - //Create blocks where higher fee/pri txs are included more often + //Create blocks where higher fee txs are included more often for (int h = 0; h <= blocknum%10; h++) { - // 10/10 blocks add highest fee/pri transactions + // 10/10 blocks add highest fee transactions // 9/10 blocks add 2nd highest and so on until ... - // 1/10 blocks add lowest fee/pri transactions + // 1/10 blocks add lowest fee transactions while (txHashes[9-h].size()) { - std::shared_ptr<const CTransaction> ptx = mpool.get(txHashes[9-h].back()); + CTransactionRef ptx = mpool.get(txHashes[9-h].back()); if (ptx) - block.push_back(*ptx); + block.push_back(ptx); txHashes[9-h].pop_back(); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); if (blocknum == 30) { // At this point we should need to combine 5 buckets to get enough data points @@ -101,7 +92,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } std::vector<CAmount> origFeeEst; - std::vector<double> origPriEst; // Highest feerate is 10*baseRate and gets in all blocks, // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, @@ -110,97 +100,88 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); - origPriEst.push_back(mpool.estimatePriority(i)); if (i > 1) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); - BOOST_CHECK(origPriEst[i-1] <= origPriEst[i-2]); } int mult = 11-i; BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); - BOOST_CHECK(origPriEst[i-1] < pow(10,mult) * basepri + deltaPri); - BOOST_CHECK(origPriEst[i-1] > pow(10,mult) * basepri - deltaPri); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change // We haven't decayed the moving average enough so we still have enough data points in every bucket while (blocknum < 250) - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] + deltaPri); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); } // Mine 15 more blocks with lots of transactions happening and not getting mined // Estimates should go up while (blocknum < 265) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee multiple + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); txHashes[j].push_back(hash); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); } int answerFound; for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) == -1 || mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); - BOOST_CHECK(mpool.estimateSmartPriority(i, &answerFound) > origPriEst[answerFound-1] - deltaPri); } // Mine all those transactions // Estimates should still not be below original for (int j = 0; j < 10; j++) { while(txHashes[j].size()) { - std::shared_ptr<const CTransaction> ptx = mpool.get(txHashes[j].back()); + CTransactionRef ptx = mpool.get(txHashes[j].back()); if (ptx) - block.push_back(*ptx); + block.push_back(ptx); txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265, dummyConflicted); + mpool.removeForBlock(block, 265); block.clear(); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); } // Mine 200 more blocks where everything is mined every block // Estimates should be below original estimates while (blocknum < 465) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee multiple + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); - std::shared_ptr<const CTransaction> ptx = mpool.get(hash); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + CTransactionRef ptx = mpool.get(hash); if (ptx) - block.push_back(*ptx); + block.push_back(ptx); + } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); } for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] - deltaPri); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee // and that estimateSmartPriority returns essentially an infinite value - mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[0][5]).Time(GetTime()).Priority(priV[1][5]).Height(blocknum).FromTx(tx, &mpool)); - // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[0][5] + mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[5]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5] mpool.TrimToSize(1); - BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[0][5]); + BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 6cad02e738..1e5de2021c 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -4,12 +4,12 @@ #include <vector> #include "prevector.h" -#include "test_random.h" #include "serialize.h" #include "streams.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index a3d1a25589..a359598ddc 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -84,6 +84,28 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_THROW(CallRPC(string("sendrawtransaction ")+rawtx+" extra"), runtime_error); } +BOOST_AUTO_TEST_CASE(rpc_togglenetwork) +{ + UniValue r; + + r = CallRPC("getnetworkinfo"); + bool netState = find_value(r.get_obj(), "networkactive").get_bool(); + BOOST_CHECK_EQUAL(netState, true); + + BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive false")); + r = CallRPC("getnetworkinfo"); + int numConnection = find_value(r.get_obj(), "connections").get_int(); + BOOST_CHECK_EQUAL(numConnection, 0); + + netState = find_value(r.get_obj(), "networkactive").get_bool(); + BOOST_CHECK_EQUAL(netState, false); + + BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive true")); + r = CallRPC("getnetworkinfo"); + netState = find_value(r.get_obj(), "networkactive").get_bool(); + BOOST_CHECK_EQUAL(netState, true); +} + BOOST_AUTO_TEST_CASE(rpc_rawsign) { UniValue r; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 561adb8ea2..3169afb13a 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -173,11 +173,14 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << tx2; - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect, message); - } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect,message); + int libconsensus_flags = flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL; + if (libconsensus_flags == flags) { + if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + } else { + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); + } } #endif } diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index bec2c7459d..bbadf57957 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -10,11 +10,54 @@ #include <stdint.h> #include <boost/test/unit_test.hpp> - using namespace std; BOOST_FIXTURE_TEST_SUITE(serialize_tests, BasicTestingSetup) +class CSerializeMethodsTestSingle +{ +protected: + int intval; + bool boolval; + std::string stringval; + const char* charstrval; + CTransaction txval; +public: + CSerializeMethodsTestSingle() = default; + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(txvalin){} + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(intval); + READWRITE(boolval); + READWRITE(stringval); + READWRITE(FLATDATA(charstrval)); + READWRITE(txval); + } + + bool operator==(const CSerializeMethodsTestSingle& rhs) + { + return intval == rhs.intval && \ + boolval == rhs.boolval && \ + stringval == rhs.stringval && \ + strcmp(charstrval, rhs.charstrval) == 0 && \ + txval == rhs.txval; + } +}; + +class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle +{ +public: + using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEMANY(intval, boolval, stringval, FLATDATA(charstrval), txval); + } +}; + BOOST_AUTO_TEST_CASE(sizes) { BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0), 0)); @@ -297,4 +340,30 @@ BOOST_AUTO_TEST_CASE(insert_delete) BOOST_CHECK_EQUAL(ss.size(), 0); } +BOOST_AUTO_TEST_CASE(class_methods) +{ + int intval(100); + bool boolval(true); + std::string stringval("testing"); + const char* charstrval("testing charstr"); + CMutableTransaction txval; + CSerializeMethodsTestSingle methodtest1(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestSingle methodtest3; + CSerializeMethodsTestMany methodtest4; + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + BOOST_CHECK(methodtest1 == methodtest2); + ss << methodtest1; + ss >> methodtest4; + ss << methodtest2; + ss >> methodtest3; + BOOST_CHECK(methodtest1 == methodtest2); + BOOST_CHECK(methodtest2 == methodtest3); + BOOST_CHECK(methodtest3 == methodtest4); + + CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, FLATDATA(charstrval), txval); + ss2 >> methodtest3; + BOOST_CHECK(methodtest3 == methodtest4); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 0b1050d020..3bc8341b02 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -6,12 +6,12 @@ #include "data/sighash.json.h" #include "hash.h" #include "main.h" // For CheckTransaction -#include "test_random.h" #include "script/interpreter.h" #include "script/script.h" #include "serialize.h" #include "streams.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index b19f8fbffb..d6835df71f 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -3,9 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chain.h" -#include "test_random.h" #include "util.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include <vector> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 34f501e867..8b715ce93e 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -15,6 +15,64 @@ using namespace boost::assign; // bring 'operator+=()' into scope BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(streams_vector_writer) +{ + unsigned char a(1); + unsigned char b(2); + unsigned char bytes[] = { 3, 4, 5, 6 }; + std::vector<unsigned char> vch; + + // Each test runs twice. Serializing a second time at the same starting + // point should yield the same results, even if the first test grew the + // vector. + + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); + vch.clear(); + + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); + vch.clear(); + + vch.resize(5, 0); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); + vch.clear(); + + vch.resize(4, 0); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); + vch.clear(); + + vch.resize(4, 0); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); + BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); + vch.clear(); + + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, FLATDATA(bytes)); + BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, FLATDATA(bytes)); + BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); + vch.clear(); + + vch.resize(4, 8); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, FLATDATA(bytes), b); + BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, FLATDATA(bytes), b); + BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); + vch.clear(); +} + BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { std::vector<char> in; diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 98f4ed939f..6cbe314a76 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -101,7 +101,7 @@ TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) { std::vector<CMutableTransaction> noTxns; CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); - coinbaseTxns.push_back(b.vtx[0]); + coinbaseTxns.push_back(*b.vtx[0]); } } @@ -119,15 +119,14 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& // Replace mempool-selected txns with just coinbase plus passed-in txns: block.vtx.resize(1); BOOST_FOREACH(const CMutableTransaction& tx, txns) - block.vtx.push_back(tx); + block.vtx.push_back(MakeTransactionRef(tx)); // IncrementExtraNonce creates a valid coinbase and merkleRoot unsigned int extraNonce = 0; IncrementExtraNonce(&block, chainActive.Tip(), extraNonce); while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; - CValidationState state; - ProcessNewBlock(state, chainparams, NULL, &block, true, NULL); + ProcessNewBlock(chainparams, &block, true, NULL, NULL); CBlock result = block; return result; @@ -138,12 +137,12 @@ TestChain100Setup::~TestChain100Setup() } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(CMutableTransaction &tx, CTxMemPool *pool) { +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx, CTxMemPool *pool) { CTransaction txn(tx); return FromTx(txn, pool); } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(CTransaction &txn, CTxMemPool *pool) { +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransaction &txn, CTxMemPool *pool) { bool hasNoDependencies = pool ? pool->HasNoInputsOf(txn) : hadNoDependencies; // Hack to assume either its completely dependent on other mempool txs or not at all CAmount inChainValue = hasNoDependencies ? txn.GetValueOut() : 0; diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 9819a7097d..3dea20445d 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -79,8 +79,8 @@ struct TestMemPoolEntryHelper nFee(0), nTime(0), dPriority(0.0), nHeight(1), hadNoDependencies(false), spendsCoinbase(false), sigOpCost(4) { } - CTxMemPoolEntry FromTx(CMutableTransaction &tx, CTxMemPool *pool = NULL); - CTxMemPoolEntry FromTx(CTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CMutableTransaction &tx, CTxMemPool *pool = NULL); + CTxMemPoolEntry FromTx(const CTransaction &tx, CTxMemPool *pool = NULL); // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index da0a3d73e0..2732948060 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -184,25 +184,25 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(OneL.begin() + 32 == OneL.end()); BOOST_CHECK(MaxL.begin() + 32 == MaxL.end()); BOOST_CHECK(TmpL.begin() + 32 == TmpL.end()); - BOOST_CHECK(R1L.GetSerializeSize(0,PROTOCOL_VERSION) == 32); - BOOST_CHECK(ZeroL.GetSerializeSize(0,PROTOCOL_VERSION) == 32); + BOOST_CHECK(GetSerializeSize(R1L, 0, PROTOCOL_VERSION) == 32); + BOOST_CHECK(GetSerializeSize(ZeroL, 0, PROTOCOL_VERSION) == 32); - std::stringstream ss; - R1L.Serialize(ss,0,PROTOCOL_VERSION); + CDataStream ss(0, PROTOCOL_VERSION); + ss << R1L; BOOST_CHECK(ss.str() == std::string(R1Array,R1Array+32)); - TmpL.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpL; BOOST_CHECK(R1L == TmpL); - ss.str(""); - ZeroL.Serialize(ss,0,PROTOCOL_VERSION); + ss.clear(); + ss << ZeroL; BOOST_CHECK(ss.str() == std::string(ZeroArray,ZeroArray+32)); - TmpL.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpL; BOOST_CHECK(ZeroL == TmpL); - ss.str(""); - MaxL.Serialize(ss,0,PROTOCOL_VERSION); + ss.clear(); + ss << MaxL; BOOST_CHECK(ss.str() == std::string(MaxArray,MaxArray+32)); - TmpL.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpL; BOOST_CHECK(MaxL == TmpL); - ss.str(""); + ss.clear(); BOOST_CHECK(R1S.GetHex() == R1S.ToString()); BOOST_CHECK(R2S.GetHex() == R2S.ToString()); @@ -230,24 +230,24 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(OneS.begin() + 20 == OneS.end()); BOOST_CHECK(MaxS.begin() + 20 == MaxS.end()); BOOST_CHECK(TmpS.begin() + 20 == TmpS.end()); - BOOST_CHECK(R1S.GetSerializeSize(0,PROTOCOL_VERSION) == 20); - BOOST_CHECK(ZeroS.GetSerializeSize(0,PROTOCOL_VERSION) == 20); + BOOST_CHECK(GetSerializeSize(R1S, 0, PROTOCOL_VERSION) == 20); + BOOST_CHECK(GetSerializeSize(ZeroS, 0, PROTOCOL_VERSION) == 20); - R1S.Serialize(ss,0,PROTOCOL_VERSION); + ss << R1S; BOOST_CHECK(ss.str() == std::string(R1Array,R1Array+20)); - TmpS.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpS; BOOST_CHECK(R1S == TmpS); - ss.str(""); - ZeroS.Serialize(ss,0,PROTOCOL_VERSION); + ss.clear(); + ss << ZeroS; BOOST_CHECK(ss.str() == std::string(ZeroArray,ZeroArray+20)); - TmpS.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpS; BOOST_CHECK(ZeroS == TmpS); - ss.str(""); - MaxS.Serialize(ss,0,PROTOCOL_VERSION); + ss.clear(); + ss << MaxS; BOOST_CHECK(ss.str() == std::string(MaxArray,MaxArray+20)); - TmpS.Unserialize(ss,0,PROTOCOL_VERSION); + ss >> TmpS; BOOST_CHECK(MaxS == TmpS); - ss.str(""); + ss.clear(); } BOOST_AUTO_TEST_CASE( conversion ) diff --git a/src/test/univalue_tests.cpp b/src/test/univalue_tests.cpp index 45d480c816..7f794fcbe9 100644 --- a/src/test/univalue_tests.cpp +++ b/src/test/univalue_tests.cpp @@ -1,4 +1,4 @@ -// Copyright 2014 BitPay, Inc. +// Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 0f1c7ab222..bad72ffc0f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,11 +6,11 @@ #include "clientversion.h" #include "primitives/transaction.h" -#include "test_random.h" #include "sync.h" #include "utilstrencodings.h" #include "utilmoneystr.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include <stdint.h> #include <vector> diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 784e796998..c05d593ed6 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -3,9 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chain.h" -#include "test_random.h" #include "versionbits.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include "chainparams.h" #include "main.h" #include "consensus/params.h" diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 1ca6b46566..ffb9993f90 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -470,7 +470,7 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& // Finally - now create the service if (private_key.empty()) // No private key, generate one - private_key = "NEW:BEST"; + private_key = "NEW:RSA1024"; // Explicitly request RSA1024 - see issue #9214 // Request hidden service, redirect port. // Note that the 'virtual' port doesn't have to be the same as our internal port, but this is just a convenient // choice. TODO; refactor the shutdown sequence some day. diff --git a/src/txdb.h b/src/txdb.h index adb3f66327..687c686775 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -43,7 +43,7 @@ struct CDiskTxPos : public CDiskBlockPos ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(*(CDiskBlockPos*)this); READWRITE(VARINT(nTxOffset)); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0f1c166abc..417a88cbef 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -24,7 +24,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, bool poolHasNoInputsOf, CAmount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): - tx(std::make_shared<CTransaction>(_tx)), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), + tx(MakeTransactionRef(_tx)), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), hadNoDependencies(poolHasNoInputsOf), inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { @@ -503,7 +503,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransaction>& removed) +void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<CTransactionRef>* removed) { // Remove transaction from memory pool { @@ -530,8 +530,10 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransact BOOST_FOREACH(txiter it, txToRemove) { CalculateDescendants(it, setAllRemoves); } - BOOST_FOREACH(txiter it, setAllRemoves) { - removed.push_back(it->GetTx()); + if (removed) { + BOOST_FOREACH(txiter it, setAllRemoves) { + removed->emplace_back(it->GetSharedTx()); + } } RemoveStaged(setAllRemoves, false); } @@ -541,7 +543,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions LOCK(cs); - list<CTransaction> transactionsToRemove; + setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction& tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); @@ -549,16 +551,16 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem if (!CheckFinalTx(tx, flags) || !CheckSequenceLocks(tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be invalid // So it's critical that we remove the tx and not depend on the LockPoints. - transactionsToRemove.push_back(tx); + txToRemove.insert(it); } else if (it->GetSpendsCoinbase()) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); + if (nCheckFrequency != 0) assert(coins); if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { - transactionsToRemove.push_back(tx); + txToRemove.insert(it); break; } } @@ -567,13 +569,14 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem mapTx.modify(it, update_lock_points(lp)); } } - BOOST_FOREACH(const CTransaction& tx, transactionsToRemove) { - list<CTransaction> removed; - removeRecursive(tx, removed); + setEntries setAllRemoves; + for (txiter it : txToRemove) { + CalculateDescendants(it, setAllRemoves); } + RemoveStaged(setAllRemoves, false); } -void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed) +void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); @@ -593,29 +596,29 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> /** * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ -void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate) +void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, + std::vector<CTransactionRef>* conflicts, bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; - BOOST_FOREACH(const CTransaction& tx, vtx) + for (const auto& tx : vtx) { - uint256 hash = tx.GetHash(); + uint256 hash = tx->GetHash(); indexed_transaction_set::iterator i = mapTx.find(hash); if (i != mapTx.end()) entries.push_back(*i); } - BOOST_FOREACH(const CTransaction& tx, vtx) + for (const auto& tx : vtx) { - txiter it = mapTx.find(tx.GetHash()); + txiter it = mapTx.find(tx->GetHash()); if (it != mapTx.end()) { setEntries stage; stage.insert(it); RemoveStaged(stage, true); } - removeConflicts(tx, conflicts); - ClearPrioritisation(tx.GetHash()); + removeConflicts(*tx, conflicts); + ClearPrioritisation(tx->GetHash()); } // After the txs in the new block have been removed from the mempool, update policy estimates minerPolicyEstimator->processBlock(nBlockHeight, entries, fCurrentEstimate); @@ -830,6 +833,10 @@ void CTxMemPool::queryHashes(vector<uint256>& vtxid) } } +static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { + return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize()), it->GetModifiedFee() - it->GetFee()}; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -838,13 +845,13 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const std::vector<TxMempoolInfo> ret; ret.reserve(mapTx.size()); for (auto it : iters) { - ret.push_back(TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize())}); + ret.push_back(GetInfo(it)); } return ret; } -std::shared_ptr<const CTransaction> CTxMemPool::get(const uint256& hash) const +CTransactionRef CTxMemPool::get(const uint256& hash) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(hash); @@ -859,7 +866,7 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) return TxMempoolInfo(); - return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; + return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const @@ -888,7 +895,7 @@ CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const { try { LOCK(cs); - fileout << 109900; // version required to read: 0.10.99 or later + fileout << 139900; // version required to read: 0.13.99 or later fileout << CLIENT_VERSION; // version that wrote the file minerPolicyEstimator->Write(fileout); } @@ -907,9 +914,8 @@ CTxMemPool::ReadFeeEstimates(CAutoFile& filein) filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein); + minerPolicyEstimator->Read(filein, nVersionThatWrote); } catch (const std::exception&) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); @@ -972,7 +978,7 @@ bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - shared_ptr<const CTransaction> ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(txid); if (ptx) { coins = CCoins(*ptx, MEMPOOL_HEIGHT); return true; diff --git a/src/txmempool.h b/src/txmempool.h index 1763930ba0..23fe5a7abe 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -6,9 +6,12 @@ #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H -#include <list> #include <memory> #include <set> +#include <map> +#include <vector> +#include <utility> +#include <string> #include "amount.h" #include "coins.h" @@ -59,7 +62,7 @@ class CTxMemPool; /** \class CTxMemPoolEntry * - * CTxMemPoolEntry stores data about the correponding transaction, as well + * CTxMemPoolEntry stores data about the corresponding transaction, as well * as data about all in-mempool transactions that depend on the transaction * ("descendant" transactions). * @@ -77,7 +80,7 @@ class CTxMemPool; class CTxMemPoolEntry { private: - std::shared_ptr<const CTransaction> tx; + CTransactionRef tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) size_t nModSize; //!< ... and modified size for priority @@ -115,7 +118,7 @@ public: CTxMemPoolEntry(const CTxMemPoolEntry& other); const CTransaction& GetTx() const { return *this->tx; } - std::shared_ptr<const CTransaction> GetSharedTx() const { return this->tx; } + CTransactionRef GetSharedTx() const { return this->tx; } /** * Fast calculation of lower bound of current priority as update * from entry priority. Only inputs that were originally in-chain will age. @@ -319,13 +322,16 @@ class CBlockPolicyEstimator; struct TxMempoolInfo { /** The transaction itself */ - std::shared_ptr<const CTransaction> tx; + CTransactionRef tx; /** Time the transaction entered the mempool. */ int64_t nTime; /** Feerate of the transaction. */ CFeeRate feeRate; + + /** The fee delta. */ + int64_t nFeeDelta; }; /** @@ -521,11 +527,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); - void removeRecursive(const CTransaction &tx, std::list<CTransaction>& removed); + void removeRecursive(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); - void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); - void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); + void removeConflicts(const CTransaction &tx, std::vector<CTransactionRef>* removed = NULL); + void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight, + std::vector<CTransactionRef>* conflicts = NULL, bool fCurrentEstimate = true); void clear(); void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); @@ -617,7 +623,7 @@ public: return (mapTx.count(hash) != 0); } - std::shared_ptr<const CTransaction> get(const uint256& hash) const; + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; diff --git a/src/ui_interface.h b/src/ui_interface.h index 177ff238db..15b9614f63 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -85,6 +85,9 @@ public: /** Number of network connections changed. */ boost::signals2::signal<void (int newNumConnections)> NotifyNumConnectionsChanged; + /** Network activity state changed. */ + boost::signals2::signal<void (bool networkActive)> NotifyNetworkActiveChanged; + /** * Status bar alerts changed. */ diff --git a/src/uint256.h b/src/uint256.h index dd8432d74c..86e7c0b6c6 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -78,11 +78,6 @@ public: return sizeof(data); } - unsigned int GetSerializeSize(int nType, int nVersion) const - { - return sizeof(data); - } - uint64_t GetUint64(int pos) const { const uint8_t* ptr = data + pos * 8; @@ -97,13 +92,13 @@ public: } template<typename Stream> - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { s.write((char*)data, sizeof(data)); } template<typename Stream> - void Unserialize(Stream& s, int nType, int nVersion) + void Unserialize(Stream& s) { s.read((char*)data, sizeof(data)); } diff --git a/src/undo.h b/src/undo.h index d4fc84c90c..a5d276e7f3 100644 --- a/src/undo.h +++ b/src/undo.h @@ -27,29 +27,23 @@ public: CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } - unsigned int GetSerializeSize(int nType, int nVersion) const { - return ::GetSerializeSize(VARINT(nHeight*2+(fCoinBase ? 1 : 0)), nType, nVersion) + - (nHeight > 0 ? ::GetSerializeSize(VARINT(this->nVersion), nType, nVersion) : 0) + - ::GetSerializeSize(CTxOutCompressor(REF(txout)), nType, nVersion); - } - template<typename Stream> - void Serialize(Stream &s, int nType, int nVersion) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0)), nType, nVersion); + void Serialize(Stream &s) const { + ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion), nType, nVersion); - ::Serialize(s, CTxOutCompressor(REF(txout)), nType, nVersion); + ::Serialize(s, VARINT(this->nVersion)); + ::Serialize(s, CTxOutCompressor(REF(txout))); } template<typename Stream> - void Unserialize(Stream &s, int nType, int nVersion) { + void Unserialize(Stream &s) { unsigned int nCode = 0; - ::Unserialize(s, VARINT(nCode), nType, nVersion); + ::Unserialize(s, VARINT(nCode)); nHeight = nCode / 2; fCoinBase = nCode & 1; if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion), nType, nVersion); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout))), nType, nVersion); + ::Unserialize(s, VARINT(this->nVersion)); + ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); } }; @@ -63,7 +57,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(vprevout); } }; @@ -77,7 +71,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(vtxundo); } }; diff --git a/src/version.h b/src/version.h index 87bd655066..87fb1a3a75 100644 --- a/src/version.h +++ b/src/version.h @@ -9,7 +9,7 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70014; +static const int PROTOCOL_VERSION = 70015; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; @@ -42,4 +42,7 @@ static const int FEEFILTER_VERSION = 70013; //! short-id-based block download starts with this version static const int SHORT_IDS_BLOCKS_VERSION = 70014; +//! not banning for invalid compact blocks starts with this version +static const int INVALID_CB_NO_BAN_VERSION = 70015; + #endif // BITCOIN_VERSION_H diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 78516770e6..08d23688ff 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -22,6 +22,8 @@ public: bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true CFeeRate nFeeRate; + //! Override the default confirmation target, 0 = use default + int nConfirmTarget; CCoinControl() { @@ -37,6 +39,7 @@ public: nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; + nConfirmTarget = 0; } bool HasSelected() const diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 190f8ecf2a..31ee060677 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -48,12 +48,12 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v int i = 0; if (nDerivationMethod == 0) - i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, chKey, chIV); + i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, vchKey.data(), vchIV.data()); if (i != (int)WALLET_CRYPTO_KEY_SIZE) { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); return false; } @@ -66,8 +66,8 @@ bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector<unsigne if (chNewKey.size() != WALLET_CRYPTO_KEY_SIZE || chNewIV.size() != WALLET_CRYPTO_IV_SIZE) return false; - memcpy(&chKey[0], &chNewKey[0], sizeof chKey); - memcpy(&chIV[0], &chNewIV[0], sizeof chIV); + memcpy(vchKey.data(), chNewKey.data(), chNewKey.size()); + memcpy(vchIV.data(), chNewIV.data(), chNewIV.size()); fKeySet = true; return true; @@ -82,7 +82,7 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned // n + AES_BLOCKSIZE bytes vchCiphertext.resize(vchPlaintext.size() + AES_BLOCKSIZE); - AES256CBCEncrypt enc(chKey, chIV, true); + AES256CBCEncrypt enc(vchKey.data(), vchIV.data(), true); size_t nLen = enc.Encrypt(&vchPlaintext[0], vchPlaintext.size(), &vchCiphertext[0]); if(nLen < vchPlaintext.size()) return false; @@ -101,7 +101,7 @@ bool CCrypter::Decrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingM vchPlaintext.resize(nLen); - AES256CBCDecrypt dec(chKey, chIV, true); + AES256CBCDecrypt dec(vchKey.data(), vchIV.data(), true); nLen = dec.Decrypt(&vchCiphertext[0], vchCiphertext.size(), &vchPlaintext[0]); if(nLen == 0) return false; diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 5d0a4a3305..e89c15b5d4 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -47,7 +47,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(vchCryptedKey); READWRITE(vchSalt); READWRITE(nDerivationMethod); @@ -77,8 +77,8 @@ class CCrypter { friend class wallet_crypto::TestCrypter; // for test access to chKey/chIV private: - unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; - unsigned char chIV[WALLET_CRYPTO_IV_SIZE]; + std::vector<unsigned char, secure_allocator<unsigned char>> vchKey; + std::vector<unsigned char, secure_allocator<unsigned char>> vchIV; bool fKeySet; int BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const; @@ -91,28 +91,21 @@ public: void CleanKey() { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); fKeySet = false; } CCrypter() { fKeySet = false; - - // Try to keep the key data out of swap (and be a bit over-careful to keep the IV that we don't even use out of swap) - // Note that this does nothing about suspend-to-disk (which will put all our key data on disk) - // Note as well that at no point in this program is any attempt made to prevent stealing of keys by reading the memory of the running process. - LockedPageManager::Instance().LockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().LockRange(&chIV[0], sizeof chIV); + vchKey.resize(WALLET_CRYPTO_KEY_SIZE); + vchIV.resize(WALLET_CRYPTO_IV_SIZE); } ~CCrypter() { CleanKey(); - - LockedPageManager::Instance().UnlockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().UnlockRange(&chIV[0], sizeof chIV); } }; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index b638810e9d..bb5337c4ad 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -798,8 +798,8 @@ UniValue processImport(const UniValue& data) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); } - std::vector<unsigned char> data(ParseHex(strPubKey)); - CPubKey pubKey(data.begin(), data.end()); + std::vector<unsigned char> vData(ParseHex(strPubKey)); + CPubKey pubKey(vData.begin(), vData.end()); if (!pubKey.IsFullyValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); @@ -1017,7 +1017,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) bool fRunScan = false; const int64_t minimumTimestamp = 1; - int64_t nLowestTimestamp; + int64_t nLowestTimestamp = 0; if (fRescan && chainActive.Tip()) { nLowestTimestamp = chainActive.Tip()->GetBlockTime(); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ab7e74988a..5a22e0278d 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -6,6 +6,7 @@ #include "amount.h" #include "base58.h" #include "chain.h" +#include "consensus/validation.h" #include "core_io.h" #include "init.h" #include "main.h" @@ -362,11 +363,14 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr vecSend.push_back(recipient); if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > pwalletMain->GetBalance()) - strError = strprintf("Error: This transaction requires a transaction fee of at least %s because of its amount, complexity, or use of recently received funds!", FormatMoney(nFeeRequired)); + strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); + throw JSONRPCError(RPC_WALLET_ERROR, strError); + } + CValidationState state; + if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get(), state)) { + strError = strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason()); throw JSONRPCError(RPC_WALLET_ERROR, strError); } - if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Error: The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of the wallet and coins were spent in the copy but not marked as spent here."); } UniValue sendtoaddress(const JSONRPCRequest& request) @@ -959,8 +963,11 @@ UniValue sendmany(const JSONRPCRequest& request) bool fCreated = pwalletMain->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); - if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction commit failed"); + CValidationState state; + if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get(), state)) { + strFailReason = strprintf("Transaction commit failed:: %s", state.GetRejectReason()); + throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + } return wtx.GetHash().GetHex(); } @@ -2279,7 +2286,7 @@ UniValue getwalletinfo(const JSONRPCRequest& request) " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index c5f55ef5f0..ce35c53c48 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -97,10 +97,10 @@ static void TestPassphraseSingle(const std::vector<unsigned char>& vchSalt, cons OldSetKeyFromPassphrase(passphrase, vchSalt, rounds, 0, chKey, chIV); - BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.chKey, sizeof(chKey)) == 0, \ - HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.chKey, crypt.chKey + (sizeof crypt.chKey))); - BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.chIV, sizeof(chIV)) == 0, \ - HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.chIV, crypt.chIV + (sizeof crypt.chIV))); + BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.vchKey.data(), crypt.vchKey.size()) == 0, \ + HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.vchKey)); + BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.vchIV.data(), crypt.vchIV.size()) == 0, \ + HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.vchIV)); if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(chKey, &correctKey[0], sizeof(chKey)) == 0, \ @@ -127,7 +127,7 @@ static void TestDecrypt(const CCrypter& crypt, const std::vector<unsigned char>& CKeyingMaterial vchDecrypted2; int result1, result2; result1 = crypt.Decrypt(vchCiphertext, vchDecrypted1); - result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.chKey, crypt.chIV); + result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); // These two should be equal. However, OpenSSL 1.0.1j introduced a change @@ -152,7 +152,7 @@ static void TestEncryptSingle(const CCrypter& crypt, const CKeyingMaterial& vchP std::vector<unsigned char> vchCiphertext2; int result1 = crypt.Encrypt(vchPlaintext, vchCiphertext1); - int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.chKey, crypt.chIV); + int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); BOOST_CHECK(vchCiphertext1 == vchCiphertext2); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index acf980c784..ecbbcb145d 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -266,7 +266,7 @@ BOOST_AUTO_TEST_CASE(coin_selection_tests) // test with many inputs for (CAmount amt=1500; amt < COIN; amt*=10) { empty_wallet(); - // Create 676 inputs (= MAX_STANDARD_TX_SIZE / 148 bytes per input) + // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) for (uint16_t j = 0; j < 676; j++) add_coin(amt); BOOST_CHECK(wallet.SelectCoinsMinConf(2000, 1, 1, vCoins, setCoinsRet, nValueRet)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 60a769704b..39c4fc3f1b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -658,8 +658,79 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { + LOCK(cs_wallet); CWalletDB walletdb(strWalletFile); - return walletdb.ReorderTransactions(this); + + // Old wallets didn't have any defined order for transactions + // Probably a bad idea to change the output of this + + // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. + typedef pair<CWalletTx*, CAccountingEntry*> TxPair; + typedef multimap<int64_t, TxPair > TxItems; + TxItems txByTime; + + for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + { + CWalletTx* wtx = &((*it).second); + txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); + } + list<CAccountingEntry> acentries; + walletdb.ListAccountCreditDebit("", acentries); + BOOST_FOREACH(CAccountingEntry& entry, acentries) + { + txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); + } + + nOrderPosNext = 0; + std::vector<int64_t> nOrderPosOffsets; + for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) + { + CWalletTx *const pwtx = (*it).second.first; + CAccountingEntry *const pacentry = (*it).second.second; + int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; + + if (nOrderPos == -1) + { + nOrderPos = nOrderPosNext++; + nOrderPosOffsets.push_back(nOrderPos); + + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + else + { + int64_t nOrderPosOff = 0; + BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) + { + if (nOrderPos >= nOffsetStart) + ++nOrderPosOff; + } + nOrderPos += nOrderPosOff; + nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); + + if (!nOrderPosOff) + continue; + + // Since we're changing the order, write it back + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + } + walletdb.WriteOrderPosNext(nOrderPosNext); + + return DB_LOAD_OK; } int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) @@ -1421,7 +1492,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) int posInBlock; for (posInBlock = 0; posInBlock < (int)block.vtx.size(); posInBlock++) { - if (AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate)) + if (AddToWalletIfInvolvingMe(*block.vtx[posInBlock], pindex, posInBlock, fUpdate)) ret++; } pindex = chainActive.Next(pindex); @@ -1463,7 +1534,8 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = *(item.second); LOCK(mempool.cs); - wtx.AcceptToMemoryPool(maxTxFee); + CValidationState state; + wtx.AcceptToMemoryPool(maxTxFee, state); } } @@ -1536,7 +1608,7 @@ CAmount CWalletTx::GetCredit(const isminefilter& filter) const if (IsCoinBase() && GetBlocksToMaturity() > 0) return 0; - int64_t credit = 0; + CAmount credit = 0; if (filter & ISMINE_SPENDABLE) { // GetBalance can assume transactions in mapWallet won't change @@ -2166,7 +2238,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt { if (nValue < 0 || recipient.nAmount < 0) { - strFailReason = _("Transaction amounts must be positive"); + strFailReason = _("Transaction amounts must not be negative"); return false; } nValue += recipient.nAmount; @@ -2174,9 +2246,9 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt if (recipient.fSubtractFeeFromAmount) nSubtractFeeFromAmount++; } - if (vecSend.empty() || nValue < 0) + if (vecSend.empty()) { - strFailReason = _("Transaction amounts must be positive"); + strFailReason = _("Transaction must have at least one recipient"); return false; } @@ -2433,17 +2505,22 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt dPriority = wtxNew.ComputePriority(dPriority, nBytes); + // Allow to override the default confirmation target over the CoinControl instance + int currentConfirmationTarget = nTxConfirmTarget; + if (coinControl && coinControl->nConfirmTarget > 0) + currentConfirmationTarget = coinControl->nConfirmTarget; + // Can we complete this as a free transaction? if (fSendFreeTransactions && nBytes <= MAX_FREE_TRANSACTION_CREATE_SIZE) { // Not enough fee: enough priority? - double dPriorityNeeded = mempool.estimateSmartPriority(nTxConfirmTarget); + double dPriorityNeeded = mempool.estimateSmartPriority(currentConfirmationTarget); // Require at least hard-coded AllowFree. if (dPriority >= dPriorityNeeded && AllowFree(dPriority)) break; } - CAmount nFeeNeeded = GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2474,7 +2551,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman) +bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state) { { LOCK2(cs_main, cs_wallet); @@ -2502,9 +2579,9 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee)) { + if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid\n"); + LogPrintf("CommitTransaction(): Error: Transaction not valid, %s\n", state.GetRejectReason()); return false; } wtxNew.RelayWalletTransaction(connman); @@ -2852,7 +2929,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() { CWalletTx *pcoin = &walletEntry.second; - if (!CheckFinalTx(*pcoin) || !pcoin->IsTrusted()) + if (!pcoin->IsTrusted()) continue; if (pcoin->IsCoinBase() && pcoin->GetBlocksToMaturity() > 0) @@ -3472,6 +3549,16 @@ bool CWallet::InitLoadWallet() return true; } +void CWallet::postInitProcess(boost::thread_group& threadGroup) +{ + // Add wallet transactions that aren't already in a block to mempool + // Do this here as mempool requires genesis block to be loaded + ReacceptWalletTransactions(); + + // Run a thread to flush wallet periodically + threadGroup.create_thread(boost::bind(&ThreadFlushWalletDB, boost::ref(this->strWalletFile))); +} + bool CWallet::ParameterInteraction() { if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) @@ -3481,6 +3568,16 @@ bool CWallet::ParameterInteraction() LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } + if (GetBoolArg("-salvagewallet", false) && SoftSetBoolArg("-rescan", true)) { + // Rewrite just private keys: rescan to find transactions + LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); + } + + // -zapwallettx implies a rescan + if (GetBoolArg("-zapwallettxes", false) && SoftSetBoolArg("-rescan", true)) { + LogPrintf("%s: parameter interaction: -zapwallettxes=<mode> -> setting -rescan=1\n", __func__); + } + if (GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); if (GetArg("-prune", 0) && GetBoolArg("-rescan", false)) @@ -3649,8 +3746,7 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee) +bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - CValidationState state; return ::AcceptToMemoryPool(mempool, state, *this, true, NULL, false, nAbsurdFee); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 3b37f7cb1f..409d817046 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -27,6 +27,7 @@ #include <vector> #include <boost/shared_ptr.hpp> +#include <boost/thread.hpp> extern CWallet* pwalletMain; @@ -53,7 +54,7 @@ static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -sendfreetransactions static const bool DEFAULT_SEND_FREE_TRANSACTIONS = false; //! -txconfirmtarget default -static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 2; +static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default static const bool DEFAULT_WALLET_RBF = false; //! Largest (in bytes) free transaction we're willing to create @@ -99,8 +100,9 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); READWRITE(nTime); READWRITE(vchPubKey); @@ -133,7 +135,7 @@ struct CRecipient typedef std::map<std::string, std::string> mapValue_t; -static void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) +static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) { if (!mapValue.count("n")) { @@ -144,7 +146,7 @@ static void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) } -static void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) +static inline void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) { if (nOrderPos == -1) return; @@ -194,7 +196,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { std::vector<uint256> vMerkleBranch; // For compatibility with older versions. READWRITE(*(CTransaction*)this); READWRITE(hashBlock); @@ -215,7 +217,7 @@ public: bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee); + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -314,7 +316,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { if (ser_action.ForRead()) Init(NULL); char fSpent = false; @@ -447,8 +449,9 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); READWRITE(vchPrivKey); READWRITE(nTimeCreated); @@ -492,8 +495,9 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); //! Note: strAccount is serialized as part of the key, not here. READWRITE(nCreditDebit); @@ -506,7 +510,7 @@ public: if (!(mapValue.empty() && _ssExtra.empty())) { - CDataStream ss(nType, nVersion); + CDataStream ss(s.GetType(), s.GetVersion()); ss.insert(ss.begin(), '\0'); ss << mapValue; ss.insert(ss.end(), _ssExtra.begin(), _ssExtra.end()); @@ -522,7 +526,7 @@ public: mapValue.clear(); if (std::string::npos != nSepPos) { - CDataStream ss(std::vector<char>(strComment.begin() + nSepPos + 1, strComment.end()), nType, nVersion); + CDataStream ss(std::vector<char>(strComment.begin() + nSepPos + 1, strComment.end()), s.GetType(), s.GetVersion()); ss >> mapValue; _ssExtra = std::vector<char>(ss.begin(), ss.end()); } @@ -596,7 +600,7 @@ public: */ mutable CCriticalSection cs_wallet; - std::string strWalletFile; + const std::string strWalletFile; void LoadKeyPool(int nIndex, const CKeyPool &keypool) { @@ -621,11 +625,9 @@ public: SetNull(); } - CWallet(const std::string& strWalletFileIn) + CWallet(const std::string& strWalletFileIn) : strWalletFile(strWalletFileIn) { SetNull(); - - strWalletFile = strWalletFileIn; fFileBacked = true; } @@ -774,7 +776,7 @@ public: */ bool CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletTx& wtxNew, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl *coinControl = NULL, bool sign = true); - bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman); + bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries); bool AddAccountingEntry(const CAccountingEntry&); @@ -912,6 +914,12 @@ public: /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ static bool InitLoadWallet(); + /** + * Wallet post-init setup + * Gives the wallet a chance to register repetitive tasks and complete post-init tasks + */ + void postInitProcess(boost::thread_group& threadGroup); + /* Wallets parameter interaction */ static bool ParameterInteraction(); @@ -979,8 +987,9 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - if (!(nType & SER_GETHASH)) + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); READWRITE(vchPubKey); } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 80bfe8255d..43fd6a20ad 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -251,82 +251,6 @@ void CWalletDB::ListAccountCreditDebit(const string& strAccount, list<CAccountin pcursor->close(); } -DBErrors CWalletDB::ReorderTransactions(CWallet* pwallet) -{ - LOCK(pwallet->cs_wallet); - // Old wallets didn't have any defined order for transactions - // Probably a bad idea to change the output of this - - // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. - typedef pair<CWalletTx*, CAccountingEntry*> TxPair; - typedef multimap<int64_t, TxPair > TxItems; - TxItems txByTime; - - for (map<uint256, CWalletTx>::iterator it = pwallet->mapWallet.begin(); it != pwallet->mapWallet.end(); ++it) - { - CWalletTx* wtx = &((*it).second); - txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); - } - list<CAccountingEntry> acentries; - ListAccountCreditDebit("", acentries); - BOOST_FOREACH(CAccountingEntry& entry, acentries) - { - txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); - } - - int64_t& nOrderPosNext = pwallet->nOrderPosNext; - nOrderPosNext = 0; - std::vector<int64_t> nOrderPosOffsets; - for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) - { - CWalletTx *const pwtx = (*it).second.first; - CAccountingEntry *const pacentry = (*it).second.second; - int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; - - if (nOrderPos == -1) - { - nOrderPos = nOrderPosNext++; - nOrderPosOffsets.push_back(nOrderPos); - - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - else - { - int64_t nOrderPosOff = 0; - BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) - { - if (nOrderPos >= nOffsetStart) - ++nOrderPosOff; - } - nOrderPos += nOrderPosOff; - nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); - - if (!nOrderPosOff) - continue; - - // Since we're changing the order, write it back - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - } - WriteOrderPosNext(nOrderPosNext); - - return DB_LOAD_OK; -} - class CWalletScanState { public: unsigned int nKeys; @@ -711,7 +635,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) WriteVersion(CLIENT_VERSION); if (wss.fAnyUnordered) - result = ReorderTransactions(pwallet); + result = pwallet->ReorderTransactions(); pwallet->laccentries.clear(); ListAccountCreditDebit("*", pwallet->laccentries); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 9c9d4922a4..eb25ac613d 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -54,7 +54,7 @@ public: CHDChain() { SetNull(); } ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(this->nVersion); READWRITE(nExternalChainCounter); @@ -93,7 +93,7 @@ public: ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(this->nVersion); READWRITE(nCreateTime); if (this->nVersion >= VERSION_WITH_HDDATA) @@ -153,6 +153,7 @@ public: /// This writes directly to the database, and will not update the CWallet's cached accounting entries! /// Use wallet.AddAccountingEntry instead, to write *and* update its caches. + bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); bool WriteAccountingEntry_Backend(const CAccountingEntry& acentry); bool ReadAccount(const std::string& strAccount, CAccount& account); bool WriteAccount(const std::string& strAccount, const CAccount& account); @@ -165,7 +166,6 @@ public: CAmount GetAccountCreditDebit(const std::string& strAccount); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& acentries); - DBErrors ReorderTransactions(CWallet* pwallet); DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(CWallet* pwallet, std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(CWallet* pwallet, std::vector<CWalletTx>& vWtx); @@ -180,7 +180,6 @@ private: CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); - bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); }; void ThreadFlushWalletDB(const std::string& strFile); |
