diff options
100 files changed, 3258 insertions, 1603 deletions
diff --git a/Makefile.am b/Makefile.am index 44fdf9c9fb..76eba51906 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,6 +39,11 @@ OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) +DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ + $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ + $(top_srcdir)/contrib/bitcoind.bash-completion \ + $(top_srcdir)/contrib/init \ + $(top_srcdir)/contrib/rpm BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py @@ -211,7 +216,7 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) @@ -49,9 +49,10 @@ lots of money. ### Automated Testing -Developers are strongly encouraged to write [unit tests](/doc/unit-tests.md) for new code, and to +Developers are strongly encouraged to write [unit tests](src/test/README.md) for new code, and to submit new unit tests for old code. Unit tests can be compiled and run -(assuming they weren't disabled in configure) with: `make check` +(assuming they weren't disabled in configure) with: `make check`. Further details on running +and extending unit tests can be found in [/src/test/README.md](/src/test/README.md). There are also [regression and integration tests](/qa) of the RPC interface, written in Python, that are run automatically on the build server. diff --git a/contrib/README.md b/contrib/README.md index ab5f57587e..4ea9700f59 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -38,7 +38,7 @@ Scripts and notes for Mac builds. RPM spec file for building bitcoin-core on RPM based distributions ### [Gitian-build](/contrib/gitian-build.sh) ### -Script for running full gitian builds. +Script for running full Gitian builds. Test and Verify Tools --------------------- diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 60fe69e7e3..6c0047833f 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -24,21 +24,64 @@ the script should be called from the git root folder as follows. ``` git diff -U0 HEAD~1.. | ./contrib/devtools/clang-format-diff.py -p1 -i -v ``` +copyright\_header.py +==================== -fix-copyright-headers.py -======================== +Provides utilities for managing copyright headers of `The Bitcoin Core +developers` in repository source files. It has three subcommands: -Every year newly updated files need to have its copyright headers updated to reflect the current year. -If you run this script from the root folder it will automatically update the year on the copyright header for all -source files if these have a git commit from the current year. +``` +$ ./copyright_header.py report <base_directory> [verbose] +$ ./copyright_header.py update <base_directory> +$ ./copyright_header.py insert <file> +``` +Running these subcommands without arguments displays a usage string. -For example a file changed in 2015 (with 2015 being the current year): +copyright\_header.py report \<base\_directory\> [verbose] +--------------------------------------------------------- -```// Copyright (c) 2009-2013 The Bitcoin Core developers``` +Produces a report of all copyright header notices found inside the source files +of a repository. Useful to quickly visualize the state of the headers. +Specifying `verbose` will list the full filenames of files of each category. -would be changed to: +copyright\_header.py update \<base\_directory\> [verbose] +--------------------------------------------------------- +Updates all the copyright headers of `The Bitcoin Core developers` which were +changed in a year more recent than is listed. For example: +``` +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers +``` +where `<lastModifiedYear>` is obtained from the `git log` history. -```// Copyright (c) 2009-2015 The Bitcoin Core developers``` +This subcommand also handles copyright headers that have only a single year. In +those cases: +``` +// Copyright (c) <year> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers +``` +where the update is appropriate. + +copyright\_header.py insert \<file\> +------------------------------------ +Inserts a copyright header for `The Bitcoin Core developers` at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has `#!` starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be `<year_introduced>-<current_year>` where +`<year_introduced>` is according to the `git log` history. If +`<year_introduced>` is equal to `<current_year>`, it will be set as a single +year rather than two hyphenated years. + +If the file already has a copyright for `The Bitcoin Core developers`, the +script will exit. gen-manpages.sh =============== diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py new file mode 100755 index 0000000000..9f35c378bf --- /dev/null +++ b/contrib/devtools/copyright_header.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import re +import fnmatch +import sys +import subprocess +import datetime +import os + +################################################################################ +# file filtering +################################################################################ + +EXCLUDE = [ + # libsecp256k1: + 'src/secp256k1/include/secp256k1.h', + 'src/secp256k1/include/secp256k1_ecdh.h', + 'src/secp256k1/include/secp256k1_recovery.h', + 'src/secp256k1/include/secp256k1_schnorr.h', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h', + # auto generated: + 'src/univalue/lib/univalue_escapes.h', + 'src/qt/bitcoinstrings.cpp', + 'src/chainparamsseeds.h', + # other external copyrights: + 'src/tinyformat.h', + 'src/leveldb/util/env_win.cc', + 'src/crypto/ctaes/bench.c', + 'qa/rpc-tests/test_framework/bignum.py', + # python init: + '*__init__.py', +] +EXCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in EXCLUDE])) + +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] +INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) + +def applies_to_file(filename): + return ((EXCLUDE_COMPILED.match(filename) is None) and + (INCLUDE_COMPILED.match(filename) is not None)) + +################################################################################ +# obtain list of files in repo according to INCLUDE and EXCLUDE +################################################################################ + +GIT_LS_CMD = 'git ls-files' + +def call_git_ls(): + out = subprocess.check_output(GIT_LS_CMD.split(' ')) + return [f for f in out.decode("utf-8").split('\n') if f != ''] + +def get_filenames_to_examine(): + filenames = call_git_ls() + return sorted([filename for filename in filenames if + applies_to_file(filename)]) + +################################################################################ +# define and compile regexes for the patterns we are looking for +################################################################################ + + +COPYRIGHT_WITH_C = 'Copyright \(c\)' +COPYRIGHT_WITHOUT_C = 'Copyright' +ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) + +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +YEAR_LIST = '(%s)(, %s)+' % (YEAR, YEAR) +ANY_YEAR_STYLE = '(%s|%s)' % (YEAR_RANGE, YEAR_LIST) +ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE, + ANY_YEAR_STYLE)) + +ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) + +def compile_copyright_regex(copyright_style, year_style, name): + return re.compile('%s %s %s' % (copyright_style, year_style, name)) + +EXPECTED_HOLDER_NAMES = [ + "Satoshi Nakamoto\n", + "The Bitcoin Core developers\n", + "The Bitcoin Core developers \n", + "Bitcoin Core Developers\n", + "the Bitcoin Core developers\n", + "The Bitcoin developers\n", + "The LevelDB Authors\. All rights reserved\.\n", + "BitPay Inc\.\n", + "BitPay, Inc\.\n", + "University of Illinois at Urbana-Champaign\.\n", + "MarcoFalke\n", + "Pieter Wuille\n", + "Pieter Wuille +\*\n", + "Pieter Wuille, Gregory Maxwell +\*\n", + "Pieter Wuille, Andrew Poelstra +\*\n", + "Andrew Poelstra +\*\n", + "Wladimir J. van der Laan\n", + "Jeff Garzik\n", + "Diederik Huys, Pieter Wuille +\*\n", + "Thomas Daede, Cory Fields +\*\n", + "Jan-Klaas Kollhof\n", + "Sam Rushing\n", + "ArtForz -- public domain half-a-node\n", +] + +DOMINANT_STYLE_COMPILED = {} +YEAR_LIST_STYLE_COMPILED = {} +WITHOUT_C_STYLE_COMPILED = {} + +for holder_name in EXPECTED_HOLDER_NAMES: + DOMINANT_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) + YEAR_LIST_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) + WITHOUT_C_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, + holder_name)) + +################################################################################ +# search file contents for copyright message of particular category +################################################################################ + +def get_count_of_copyrights_of_any_style_any_holder(contents): + return len(ANY_COPYRIGHT_COMPILED.findall(contents)) + +def file_has_dominant_style_copyright_for_holder(contents, holder_name): + match = DOMINANT_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_year_list_style_copyright_for_holder(contents, holder_name): + match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_without_c_style_copyright_for_holder(contents, holder_name): + match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +################################################################################ +# get file info +################################################################################ + +def read_file(filename): + return open(os.path.abspath(filename), 'r').read() + +def gather_file_info(filename): + info = {} + info['filename'] = filename + c = read_file(filename) + info['contents'] = c + + info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c) + + info['classified_copyrights'] = 0 + info['dominant_style'] = {} + info['year_list_style'] = {} + info['without_c_style'] = {} + for holder_name in EXPECTED_HOLDER_NAMES: + has_dominant_style = ( + file_has_dominant_style_copyright_for_holder(c, holder_name)) + has_year_list_style = ( + file_has_year_list_style_copyright_for_holder(c, holder_name)) + has_without_c_style = ( + file_has_without_c_style_copyright_for_holder(c, holder_name)) + info['dominant_style'][holder_name] = has_dominant_style + info['year_list_style'][holder_name] = has_year_list_style + info['without_c_style'][holder_name] = has_without_c_style + if has_dominant_style or has_year_list_style or has_without_c_style: + info['classified_copyrights'] = info['classified_copyrights'] + 1 + return info + +################################################################################ +# report execution +################################################################################ + +SEPARATOR = '-'.join(['' for _ in range(80)]) + +def print_filenames(filenames, verbose): + if not verbose: + return + for filename in filenames: + print("\t%s" % filename) + +def print_report(file_infos, verbose): + print(SEPARATOR) + examined = [i['filename'] for i in file_infos] + print("%d files examined according to INCLUDE and EXCLUDE fnmatch rules" % + len(examined)) + print_filenames(examined, verbose) + + print(SEPARATOR) + print('') + zero_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 0] + print("%4d with zero copyrights" % len(zero_copyrights)) + print_filenames(zero_copyrights, verbose) + one_copyright = [i['filename'] for i in file_infos if + i['all_copyrights'] == 1] + print("%4d with one copyright" % len(one_copyright)) + print_filenames(one_copyright, verbose) + two_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 2] + print("%4d with two copyrights" % len(two_copyrights)) + print_filenames(two_copyrights, verbose) + three_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 3] + print("%4d with three copyrights" % len(three_copyrights)) + print_filenames(three_copyrights, verbose) + four_or_more_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] >= 4] + print("%4d with four or more copyrights" % len(four_or_more_copyrights)) + print_filenames(four_or_more_copyrights, verbose) + print('') + print(SEPARATOR) + print('Copyrights with dominant style:\ne.g. "Copyright (c)" and ' + '"<year>" or "<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + dominant_style = [i['filename'] for i in file_infos if + i['dominant_style'][holder_name]] + if len(dominant_style) > 0: + print("%4d with '%s'" % (len(dominant_style), + holder_name.replace('\n', '\\n'))) + print_filenames(dominant_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with year list style:\ne.g. "Copyright (c)" and ' + '"<year1>, <year2>, ...":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + year_list_style = [i['filename'] for i in file_infos if + i['year_list_style'][holder_name]] + if len(year_list_style) > 0: + print("%4d with '%s'" % (len(year_list_style), + holder_name.replace('\n', '\\n'))) + print_filenames(year_list_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "<year>" or ' + '"<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + without_c_style = [i['filename'] for i in file_infos if + i['without_c_style'][holder_name]] + if len(without_c_style) > 0: + print("%4d with '%s'" % (len(without_c_style), + holder_name.replace('\n', '\\n'))) + print_filenames(without_c_style, verbose) + + print('') + print(SEPARATOR) + + unclassified_copyrights = [i['filename'] for i in file_infos if + i['classified_copyrights'] < i['all_copyrights']] + print("%d with unexpected copyright holder names" % + len(unclassified_copyrights)) + print_filenames(unclassified_copyrights, verbose) + print(SEPARATOR) + +def exec_report(base_directory, verbose): + original_cwd = os.getcwd() + os.chdir(base_directory) + filenames = get_filenames_to_examine() + file_infos = [gather_file_info(f) for f in filenames] + print_report(file_infos, verbose) + os.chdir(original_cwd) + +################################################################################ +# report cmd +################################################################################ + +REPORT_USAGE = """ +Produces a report of all copyright header notices found inside the source files +of a repository. + +Usage: + $ ./copyright_header.py report <base_directory> [verbose] + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. + [verbose] - Includes a list of every file of each subcategory in the report. +""" + +def report_cmd(argv): + if len(argv) == 2: + sys.exit(REPORT_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad <base_directory>: %s" % base_directory) + + if len(argv) == 3: + verbose = False + elif argv[3] == 'verbose': + verbose = True + else: + sys.exit("*** unknown argument: %s" % argv[2]) + + exec_report(base_directory, verbose) + +################################################################################ +# query git for year of last change +################################################################################ + +GIT_LOG_CMD = "git log --pretty=format:%%ai %s" + +def call_git_log(filename): + out = subprocess.check_output((GIT_LOG_CMD % filename).split(' ')) + return out.decode("utf-8").split('\n') + +def get_git_change_years(filename): + git_log_lines = call_git_log(filename) + if len(git_log_lines) == 0: + return [datetime.date.today().year] + # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" + return [line.split(' ')[0].split('-')[0] for line in git_log_lines] + +def get_most_recent_git_change_year(filename): + return max(get_git_change_years(filename)) + +################################################################################ +# read and write to file +################################################################################ + +def read_file_lines(filename): + f = open(os.path.abspath(filename), 'r') + file_lines = f.readlines() + f.close() + return file_lines + +def write_file_lines(filename, file_lines): + f = open(os.path.abspath(filename), 'w') + f.write(''.join(file_lines)) + f.close() + +################################################################################ +# update header years execution +################################################################################ + +COPYRIGHT = 'Copyright \(c\)' +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +HOLDER = 'The Bitcoin Core developers' +UPDATEABLE_LINE_COMPILED = re.compile(' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) + +def get_updatable_copyright_line(file_lines): + index = 0 + for line in file_lines: + if UPDATEABLE_LINE_COMPILED.search(line) is not None: + return index, line + index = index + 1 + return None, None + +def parse_year_range(year_range): + year_split = year_range.split('-') + start_year = year_split[0] + if len(year_split) == 1: + return start_year, start_year + return start_year, year_split[1] + +def year_range_to_str(start_year, end_year): + if start_year == end_year: + return start_year + return "%s-%s" % (start_year, end_year) + +def create_updated_copyright_line(line, last_git_change_year): + copyright_splitter = 'Copyright (c) ' + copyright_split = line.split(copyright_splitter) + # Preserve characters on line that are ahead of the start of the copyright + # notice - they are part of the comment block and vary from file-to-file. + before_copyright = copyright_split[0] + after_copyright = copyright_split[1] + + space_split = after_copyright.split(' ') + year_range = space_split[0] + start_year, end_year = parse_year_range(year_range) + if end_year == last_git_change_year: + return line + return (before_copyright + copyright_splitter + + year_range_to_str(start_year, last_git_change_year) + ' ' + + ' '.join(space_split[1:])) + +def update_updatable_copyright(filename): + file_lines = read_file_lines(filename) + index, line = get_updatable_copyright_line(file_lines) + if not line: + print_file_action_message(filename, "No updatable copyright.") + return + last_git_change_year = get_most_recent_git_change_year(filename) + new_line = create_updated_copyright_line(line, last_git_change_year) + if line == new_line: + print_file_action_message(filename, "Copyright up-to-date.") + return + file_lines[index] = new_line + write_file_lines(filename, file_lines) + print_file_action_message(filename, + "Copyright updated! -> %s" % last_git_change_year) + +def exec_update_header_year(base_directory): + original_cwd = os.getcwd() + os.chdir(base_directory) + for filename in get_filenames_to_examine(): + update_updatable_copyright(filename) + os.chdir(original_cwd) + +################################################################################ +# update cmd +################################################################################ + +UPDATE_USAGE = """ +Updates all the copyright headers of "The Bitcoin Core developers" which were +changed in a year more recent than is listed. For example: + +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers + +where <lastModifiedYear> is obtained from the 'git log' history. + +This subcommand also handles copyright headers that have only a single year. In those cases: + +// Copyright (c) <year> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers + +where the update is appropriate. + +Usage: + $ ./copyright_header.py update <base_directory> + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. +""" + +def print_file_action_message(filename, action): + print("%-52s %s" % (filename, action)) + +def update_cmd(argv): + if len(argv) != 3: + sys.exit(UPDATE_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad base_directory: %s" % base_directory) + exec_update_header_year(base_directory) + +################################################################################ +# inserted copyright header format +################################################################################ + +def get_header_lines(header, start_year, end_year): + lines = header.split('\n')[1:-1] + lines[0] = lines[0] % year_range_to_str(start_year, end_year) + return [line + '\n' for line in lines] + +CPP_HEADER = ''' +// Copyright (c) %s The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_cpp_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) + +PYTHON_HEADER = ''' +# Copyright (c) %s The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_python_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) + +################################################################################ +# query git for year of last change +################################################################################ + +def get_git_change_year_range(filename): + years = get_git_change_years(filename) + return min(years), max(years) + +################################################################################ +# check for existing core copyright +################################################################################ + +def file_already_has_core_copyright(file_lines): + index, _ = get_updatable_copyright_line(file_lines) + return index != None + +################################################################################ +# insert header execution +################################################################################ + +def file_has_hashbang(file_lines): + if len(file_lines) < 1: + return False + if len(file_lines[0]) <= 2: + return False + return file_lines[0][:2] == '#!' + +def insert_python_header(filename, file_lines, start_year, end_year): + if file_has_hashbang(file_lines): + insert_idx = 1 + else: + insert_idx = 0 + header_lines = get_python_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(insert_idx, line) + write_file_lines(filename, file_lines) + +def insert_cpp_header(filename, file_lines, start_year, end_year): + header_lines = get_cpp_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(0, line) + write_file_lines(filename, file_lines) + +def exec_insert_header(filename, style): + file_lines = read_file_lines(filename) + if file_already_has_core_copyright(file_lines): + sys.exit('*** %s already has a copyright by The Bitcoin Core developers' + % (filename)) + start_year, end_year = get_git_change_year_range(filename) + if style == 'python': + insert_python_header(filename, file_lines, start_year, end_year) + else: + insert_cpp_header(filename, file_lines, start_year, end_year) + +################################################################################ +# insert cmd +################################################################################ + +INSERT_USAGE = """ +Inserts a copyright header for "The Bitcoin Core developers" at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has a '#!' starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be: + +"<year_introduced>-<current_year>" + +where <year_introduced> is according to the 'git log' history. If +<year_introduced> is equal to <current_year>, the date will be set to be: + +"<current_year>" + +If the file already has a copyright for "The Bitcoin Core developers", the +script will exit. + +Usage: + $ ./copyright_header.py insert <file> + +Arguments: + <file> - A source file in the bitcoin repository. +""" + +def insert_cmd(argv): + if len(argv) != 3: + sys.exit(INSERT_USAGE) + + filename = argv[2] + if not os.path.isfile(filename): + sys.exit("*** bad filename: %s" % filename) + _, extension = os.path.splitext(filename) + if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: + sys.exit("*** cannot insert for file extension %s" % extension) + + if extension == '.py': + style = 'python' + else: + style = 'cpp' + exec_insert_header(filename, style) + +################################################################################ +# UI +################################################################################ + +USAGE = """ +copyright_header.py - utilities for managing copyright headers of 'The Bitcoin +Core developers' in repository source files. + +Usage: + $ ./copyright_header <subcommand> + +Subcommands: + report + update + insert + +To see subcommand usage, run them without arguments. +""" + +SUBCOMMANDS = ['report', 'update', 'insert'] + +if __name__ == "__main__": + if len(sys.argv) == 1: + sys.exit(USAGE) + subcommand = sys.argv[1] + if subcommand not in SUBCOMMANDS: + sys.exit(USAGE) + if subcommand == 'report': + report_cmd(sys.argv) + elif subcommand == 'update': + update_cmd(sys.argv) + elif subcommand == 'insert': + insert_cmd(sys.argv) diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py deleted file mode 100755 index 54836bd83f..0000000000 --- a/contrib/devtools/fix-copyright-headers.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -""" -Run this script to update all the copyright headers of files -that were changed this year. - -For example: - -// Copyright (c) 2009-2012 The Bitcoin Core developers - -it will change it to - -// Copyright (c) 2009-2015 The Bitcoin Core developers -""" -import subprocess -import time -import re - -CMD_GIT_LIST_FILES = ['git', 'ls-files'] -CMD_GIT_DATE = ['git', 'log', '--format=%ad', '--date=short', '-1'] -CMD_PERL_REGEX = ['perl', '-pi', '-e'] -REGEX_TEMPLATE = 's/(20\\d\\d)(?:-20\\d\\d)? The Bitcoin/$1-%s The Bitcoin/' - -FOLDERS = ["qa/", "src/"] -EXTENSIONS = [".cpp",".h", ".py"] - - -def get_git_date(file_path): - d = subprocess.run(CMD_GIT_DATE + [file_path], - stdout=subprocess.PIPE, - check=True, - universal_newlines=True).stdout - # yyyy-mm-dd - return d.split('-')[0] - - -def skip_file(file_path): - for ext in EXTENSIONS: - if file_path.endswith(ext): - return False - else: - return True - -if __name__ == "__main__": - year = str(time.gmtime()[0]) - regex_current = re.compile("%s The Bitcoin" % year) - n = 1 - for folder in FOLDERS: - for file_path in subprocess.run( - CMD_GIT_LIST_FILES + [folder], - stdout=subprocess.PIPE, - check=True, - universal_newlines=True - ).stdout.split("\n"): - if skip_file(file_path): - # print(file_path, "(skip)") - continue - git_date = get_git_date(file_path) - if not year == git_date: - # print(file_path, year, "(skip)") - continue - if regex_current.search(open(file_path, "r").read()) is not None: - # already up to date - # print(file_path, year, "(skip)") - continue - print(n, file_path, "(update to %s)" % year) - subprocess.run(CMD_PERL_REGEX + [REGEX_TEMPLATE % year, file_path], check=True) - n = n + 1 diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index f82362fe41..aae966a8f6 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 Bitcoin Core Developers +# Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 32b57b3160..fe01b5b957 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -27,7 +27,7 @@ remotes: files: [] script: | WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" diff --git a/doc/README.md b/doc/README.md index e4fa49614a..8b9c0ea262 100644 --- a/doc/README.md +++ b/doc/README.md @@ -53,7 +53,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) -- [Unit Tests](unit-tests.md) - [Travis CI](travis-ci.md) - [Unauthenticated REST Interface](REST-interface.md) - [Shared Libraries](shared-libraries.md) diff --git a/doc/build-osx.md b/doc/build-osx.md index bc90a30562..63a7ee28ca 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -90,6 +90,6 @@ Uncheck everything except Qt Creator during the installation process. Notes ----- -* Tested on OS X 10.7 through 10.11 on 64-bit Intel processors only. +* Tested on OS X 10.8 through 10.12 on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) diff --git a/doc/files.md b/doc/files.md index f7eca57dcb..928977143b 100644 --- a/doc/files.md +++ b/doc/files.md @@ -10,6 +10,7 @@ * db.log: wallet database log file * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 +* mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 * wallet.dat: personal wallet (BDB) with keys and transactions * .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown): since 0.12.0 diff --git a/doc/release-notes.md b/doc/release-notes.md index 0463cb8a61..f511fee22e 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -48,6 +48,15 @@ Low-level RPC changes an optional third arg, which was always ignored. Make sure to never pass more than two arguments. +Removal of Priority Estimation +------------------------------ + +- Estimation of "priority" needed for a transaction to be included within a target + number of blocks has been removed. The rpc calls are deprecated and will either + return -1 or 1e24 appropriately. The format for fee_estimates.dat has also + changed to no longer save these priority estimates. It will automatically be + converted to the new format which is not readable by prior versions of the + software. 0.14.0 Change log ================= diff --git a/doc/release-notes/release-notes-0.13.1.md b/doc/release-notes/release-notes-0.13.1.md new file mode 100644 index 0000000000..75c2d61be8 --- /dev/null +++ b/doc/release-notes/release-notes-0.13.1.md @@ -0,0 +1,410 @@ +Bitcoin Core version 0.13.1 is now available from: + + <https://bitcoin.org/bin/bitcoin-core-0.13.1/> + +This is a new minor version release, including activation parameters for the +segwit softfork, various bugfixes and performance improvements, as well as +updated translations. + +Please report bugs using the issue tracker at github: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +Compatibility +============== + +Microsoft ended support for Windows XP on [April 8th, 2014](https://www.microsoft.com/en-us/WindowsForBusiness/end-of-xp-support), +an OS initially released in 2001. This means that not even critical security +updates will be released anymore. Without security updates, using a bitcoin +wallet on a XP machine is irresponsible at least. + +In addition to that, with 0.12.x there have been varied reports of Bitcoin Core +randomly crashing on Windows XP. It is [not clear](https://github.com/bitcoin/bitcoin/issues/7681#issuecomment-217439891) +what the source of these crashes is, but it is likely that upstream +libraries such as Qt are no longer being tested on XP. + +We do not have time nor resources to provide support for an OS that is +end-of-life. From 0.13.0 on, Windows XP is no longer supported. Users are +suggested to upgrade to a newer version of Windows, or install an alternative OS +that is supported. + +No attempt is made to prevent installing or running the software on Windows XP, +you can still do so at your own risk, but do not expect it to work: do not +report issues about Windows XP to the issue tracker. + +From 0.13.1 onwards OS X 10.7 is no longer supported. 0.13.0 was intended to work on 10.7+, +but severe issues with the libc++ version on 10.7.x keep it from running reliably. +0.13.1 now requires 10.8+, and will communicate that to 10.7 users, rather than crashing unexpectedly. + +Notable changes +=============== + +Segregated witness soft fork +---------------------------- + +Segregated witness (segwit) is a soft fork that, if activated, will +allow transaction-producing software to separate (segregate) transaction +signatures (witnesses) from the part of the data in a transaction that is +covered by the txid. This provides several immediate benefits: + +- **Elimination of unwanted transaction malleability:** Segregating the witness + allows both existing and upgraded software to calculate the transaction + identifier (txid) of transactions without referencing the witness, which can + sometimes be changed by third-parties (such as miners) or by co-signers in a + multisig spend. This solves all known cases of unwanted transaction + malleability, which is a problem that makes programming Bitcoin wallet + software more difficult and which seriously complicates the design of smart + contracts for Bitcoin. + +- **Capacity increase:** Segwit transactions contain new fields that are not + part of the data currently used to calculate the size of a block, which + allows a block containing segwit transactions to hold more data than allowed + by the current maximum block size. Estimates based on the transactions + currently found in blocks indicate that if all wallets switch to using + segwit, the network will be able to support about 70% more transactions. The + network will also be able to support more of the advanced-style payments + (such as multisig) than it can support now because of the different weighting + given to different parts of a transaction after segwit activates (see the + following section for details). + +- **Weighting data based on how it affects node performance:** Some parts of + each Bitcoin block need to be stored by nodes in order to validate future + blocks; other parts of a block can be immediately forgotten (pruned) or used + only for helping other nodes sync their copy of the block chain. One large + part of the immediately prunable data are transaction signatures (witnesses), + and segwit makes it possible to give a different "weight" to segregated + witnesses to correspond with the lower demands they place on node resources. + Specifically, each byte of a segregated witness is given a weight of 1, each + other byte in a block is given a weight of 4, and the maximum allowed weight + of a block is 4 million. Weighting the data this way better aligns the most + profitable strategy for creating blocks with the long-term costs of block + validation. + +- **Signature covers value:** A simple improvement in the way signatures are + generated in segwit simplifies the design of secure signature generators + (such as hardware wallets), reduces the amount of data the signature + generator needs to download, and allows the signature generator to operate + more quickly. This is made possible by having the generator sign the amount + of bitcoins they think they are spending, and by having full nodes refuse to + accept those signatures unless the amount of bitcoins being spent is exactly + the same as was signed. For non-segwit transactions, wallets instead had to + download the complete previous transactions being spent for every payment + they made, which could be a slow operation on hardware wallets and in other + situations where bandwidth or computation speed was constrained. + +- **Linear scaling of sighash operations:** In 2015 a block was produced that + required about 25 seconds to validate on modern hardware because of the way + transaction signature hashes are performed. Other similar blocks, or blocks + that could take even longer to validate, can still be produced today. The + problem that caused this can't be fixed in a soft fork without unwanted + side-effects, but transactions that opt-in to using segwit will now use a + different signature method that doesn't suffer from this problem and doesn't + have any unwanted side-effects. + +- **Increased security for multisig:** Bitcoin addresses (both P2PKH addresses + that start with a '1' and P2SH addresses that start with a '3') use a hash + function known as RIPEMD-160. For P2PKH addresses, this provides about 160 + bits of security---which is beyond what cryptographers believe can be broken + today. But because P2SH is more flexible, only about 80 bits of security is + provided per address. Although 80 bits is very strong security, it is within + the realm of possibility that it can be broken by a powerful adversary. + Segwit allows advanced transactions to use the SHA256 hash function instead, + which provides about 128 bits of security (that is 281 trillion times as + much security as 80 bits and is equivalent to the maximum bits of security + believed to be provided by Bitcoin's choice of parameters for its Elliptic + Curve Digital Security Algorithm [ECDSA].) + +- **More efficient almost-full-node security** Satoshi Nakamoto's original + Bitcoin paper describes a method for allowing newly-started full nodes to + skip downloading and validating some data from historic blocks that are + protected by large amounts of proof of work. Unfortunately, Nakamoto's + method can't guarantee that a newly-started node using this method will + produce an accurate copy of Bitcoin's current ledger (called the UTXO set), + making the node vulnerable to falling out of consensus with other nodes. + Although the problems with Nakamoto's method can't be fixed in a soft fork, + Segwit accomplishes something similar to his original proposal: it makes it + possible for a node to optionally skip downloading some blockchain data + (specifically, the segregated witnesses) while still ensuring that the node + can build an accurate copy of the UTXO set for the block chain with the most + proof of work. Segwit enables this capability at the consensus layer, but + note that Bitcoin Core does not provide an option to use this capability as + of this 0.13.1 release. + +- **Script versioning:** Segwit makes it easy for future soft forks to allow + Bitcoin users to individually opt-in to almost any change in the Bitcoin + Script language when those users receive new transactions. Features + currently being researched by Bitcoin Core contributors that may use this + capability include support for Schnorr signatures, which can improve the + privacy and efficiency of multisig transactions (or transactions with + multiple inputs), and Merklized Abstract Syntax Trees (MAST), which can + improve the privacy and efficiency of scripts with two or more conditions. + Other Bitcoin community members are studying several other improvements + that can be made using script versioning. + +Activation for the segwit soft fork is being managed using BIP9 +versionbits. Segwit's version bit is bit 1, and nodes will begin +tracking which blocks signal support for segwit at the beginning of the +first retarget period after segwit's start date of 15 November 2016. If +95% of blocks within a 2,016-block retarget period (about two weeks) +signal support for segwit, the soft fork will be locked in. After +another 2,016 blocks, segwit will activate. + +For more information about segwit, please see the [segwit FAQ][], the +[segwit wallet developers guide][] or BIPs [141][BIP141], [143][BIP143], +[144][BIP144], and [145][BIP145]. If you're a miner or mining pool +operator, please see the [versionbits FAQ][] for information about +signaling support for a soft fork. + +[Segwit FAQ]: https://bitcoincore.org/en/2016/01/26/segwit-benefits/ +[segwit wallet developers guide]: https://bitcoincore.org/en/segwit_wallet_dev/ +[BIP141]: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki +[BIP143]: https://github.com/bitcoin/bips/blob/master/bip-0143.mediawiki +[BIP144]: https://github.com/bitcoin/bips/blob/master/bip-0144.mediawiki +[BIP145]: https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki +[versionbits FAQ]: https://bitcoincore.org/en/2016/06/08/version-bits-miners-faq/ + + +Null dummy soft fork +------------------- + +Combined with the segwit soft fork is an additional change that turns a +long-existing network relay policy into a consensus rule. The +`OP_CHECKMULTISIG` and `OP_CHECKMULTISIGVERIFY` opcodes consume an extra +stack element ("dummy element") after signature validation. The dummy +element is not inspected in any manner, and could be replaced by any +value without invalidating the script. + +Because any value can be used for this dummy element, it's possible for +a third-party to insert data into other people's transactions, changing +the transaction's txid (called transaction malleability) and possibly +causing other problems. + +Since Bitcoin Core 0.10.0, nodes have defaulted to only relaying and +mining transactions whose dummy element was a null value (0x00, also +called OP_0). The null dummy soft fork turns this relay rule into a +consensus rule both for non-segwit transactions and segwit transactions, +so that this method of mutating transactions is permanently eliminated +from the network. + +Signaling for the null dummy soft fork is done by signaling support +for segwit, and the null dummy soft fork will activate at the same time +as segwit. + +For more information, please see [BIP147][]. + +[BIP147]: https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki + +Low-level RPC changes +--------------------- + +- `importprunedfunds` only accepts two required arguments. Some versions accept + an optional third arg, which was always ignored. Make sure to never pass more + than two arguments. + + +Linux ARM builds +---------------- + +With the 0.13.0 release, pre-built Linux ARM binaries were added to the set of +uploaded executables. Additional detail on the ARM architecture targeted by each +is provided below. + +The following extra files can be found in the download directory or torrent: + +- `bitcoin-${VERSION}-arm-linux-gnueabihf.tar.gz`: Linux binaries targeting + the 32-bit ARMv7-A architecture. +- `bitcoin-${VERSION}-aarch64-linux-gnu.tar.gz`: Linux binaries targeting + the 64-bit ARMv8-A architecture. + +ARM builds are still experimental. If you have problems on a certain device or +Linux distribution combination please report them on the bug tracker, it may be +possible to resolve them. Note that the device you use must be (backward) +compatible with the architecture targeted by the binary that you use. +For example, a Raspberry Pi 2 Model B or Raspberry Pi 3 Model B (in its 32-bit +execution state) device, can run the 32-bit ARMv7-A targeted binary. However, +no model of Raspberry Pi 1 device can run either binary because they are all +ARMv6 architecture devices that are not compatible with ARMv7-A or ARMv8-A. + +Note that Android is not considered ARM Linux in this context. The executables +are not expected to work out of the box on Android. + + +0.13.1 Change log +================= + +Detailed release notes follow. This overview includes changes that affect +behavior, not code moves, refactors and string updates. For convenience in locating +the code changes and accompanying discussion, both the pull request and +git merge commit are mentioned. + +### Consensus +- #8636 `9dfa0c8` Implement NULLDUMMY softfork (BIP147) (jl2012) +- #8848 `7a34a46` Add NULLDUMMY verify flag in bitcoinconsensus.h (jl2012) +- #8937 `8b66659` Define start and end time for segwit deployment (sipa) + +### RPC and other APIs +- #8581 `526d2b0` Drop misleading option in importprunedfunds (MarcoFalke) +- #8699 `a5ec248` Remove createwitnessaddress RPC command (jl2012) +- #8780 `794b007` Deprecate getinfo (MarcoFalke) +- #8832 `83ad563` Throw JSONRPCError when utxo set can not be read (MarcoFalke) +- #8884 `b987348` getblockchaininfo help: pruneheight is the lowest, not highest, block (luke-jr) +- #8858 `3f508ed` rpc: Generate auth cookie in hex instead of base64 (laanwj) +- #8951 `7c2bf4b` RPC/Mining: getblocktemplate: Update and fix formatting of help (luke-jr) + +### Block and transaction handling +- #8611 `a9429ca` Reduce default number of blocks to check at startup (sipa) +- #8634 `3e80ab7` Add policy: null signature for failed CHECK(MULTI)SIG (jl2012) +- #8525 `1672225` Do not store witness txn in rejection cache (sipa) +- #8499 `9777fe1` Add several policy limits and disable uncompressed keys for segwit scripts (jl2012) +- #8526 `0027672` Make non-minimal OP_IF/NOTIF argument non-standard for P2WSH (jl2012) +- #8524 `b8c79a0` Precompute sighashes (sipa) +- #8651 `b8c79a0` Predeclare PrecomputedTransactionData as struct (sipa) + +### P2P protocol and network code +- #8740 `42ea51a` No longer send local address in addrMe (laanwj) +- #8427 `69d1cd2` Ignore `notfound` P2P messages (laanwj) +- #8573 `4f84082` Set jonasschnellis dns-seeder filter flag (jonasschnelli) +- #8712 `23feab1` Remove maxuploadtargets recommended minimum (jonasschnelli) +- #8862 `7ae6242` Fix a few cases where messages were sent after requested disconnect (theuni) +- #8393 `fe1975a` Support for compact blocks together with segwit (sipa) +- #8282 `2611ad7` Feeler connections to increase online addrs in the tried table (EthanHeilman) +- #8612 `2215c22` Check for compatibility with download in FindNextBlocksToDownload (sipa) +- #8606 `bbf379b` Fix some locks (sipa) +- #8594 `ab295bb` Do not add random inbound peers to addrman (gmaxwell) +- #8940 `5b4192b` Add x9 service bit support to dnsseed.bluematt.me, seed.bitcoinstats.com (TheBlueMatt, cdecker) +- #8944 `685e4c7` Remove bogus assert on number of oubound connections. (TheBlueMatt) +- #8949 `0dbc48a` Be more agressive in getting connections to peers with relevant services (gmaxwell) + +### Build system +- #8293 `fa5b249` Allow building libbitcoinconsensus without any univalue (luke-jr) +- #8492 `8b0bdd3` Allow building bench_bitcoin by itself (luke-jr) +- #8563 `147003c` Add configure check for -latomic (ajtowns) +- #8626 `ea51b0f` Berkeley DB v6 compatibility fix (netsafe) +- #8520 `75f2065` Remove check for `openssl/ec.h` (laanwj) + +### GUI +- #8481 `d9f0d4e` Fix minimize and close bugs (adlawren) +- #8487 `a37cec5` Persist the datadir after option reset (achow101) +- #8697 `41fd852` Fix op order to append first alert (rodasmith) +- #8678 `8e03382` Fix UI bug that could result in paying unexpected fee (jonasschnelli) +- #8911 `7634d8e` Translate all files, even if wallet disabled (laanwj) +- #8540 `1db3352` Fix random segfault when closing "Choose data directory" dialog (laanwj) +- #7579 `f1c0d78` Show network/chain errors in the GUI (jonasschnelli) + +### Wallet +- #8443 `464dedd` Trivial cleanup of HD wallet changes (jonasschnelli) +- #8539 `cb07f19` CDB: fix debug output (crowning-) +- #8664 `091cdeb` Fix segwit-related wallet bug (sdaftuar) +- #8693 `c6a6291` Add witness address to address book (instagibbs) +- #8765 `6288659` Remove "unused" ThreadFlushWalletDB from removeprunedfunds (jonasschnelli) + +### Tests and QA +- #8713 `ae8c7df` create_cache: Delete temp dir when done (MarcoFalke) +- #8716 `e34374e` Check legacy wallet as well (MarcoFalke) +- #8750 `d6ebe13` Refactor RPCTestHandler to prevent TimeoutExpired (MarcoFalke) +- #8652 `63462c2` remove root test directory for RPC tests (yurizhykin) +- #8724 `da94272` walletbackup: Sync blocks inside the loop (MarcoFalke) +- #8400 `bea02dc` enable rpcbind_test (yurizhykin) +- #8417 `f70be14` Add walletdump RPC test (including HD- & encryption-tests) (jonasschnelli) +- #8419 `a7aa3cc` Enable size accounting in mining unit tests (sdaftuar) +- #8442 `8bb1efd` Rework hd wallet dump test (MarcoFalke) +- #8528 `3606b6b` Update p2p-segwit.py to reflect correct behavior (instagibbs) +- #8531 `a27cdd8` abandonconflict: Use assert_equal (MarcoFalke) +- #8667 `6b07362` Fix SIGHASH_SINGLE bug in test_framework SignatureHash (jl2012) +- #8673 `03b0196` Fix obvious assignment/equality error in test (JeremyRubin) +- #8739 `cef633c` Fix broken sendcmpct test in p2p-compactblocks.py (sdaftuar) +- #8418 `ff893aa` Add tests for compact blocks (sdaftuar) +- #8803 `375437c` Ping regularly in p2p-segwit.py to keep connection alive (jl2012) +- #8827 `9bbe66e` Split up slow RPC calls to avoid pruning test timeouts (sdaftuar) +- #8829 `2a8bca4` Add bitcoin-tx JSON tests (jnewbery) +- #8834 `1dd1783` blockstore: Switch to dumb dbm (MarcoFalke) +- #8835 `d87227d` nulldummy.py: Don't run unused code (MarcoFalke) +- #8836 `eb18cc1` bitcoin-util-test.py should fail if the output file is empty (jnewbery) +- #8839 `31ab2f8` Avoid ConnectionResetErrors during RPC tests (laanwj) +- #8840 `cbc3fe5` Explicitly set encoding to utf8 when opening text files (laanwj) +- #8841 `3e4abb5` Fix nulldummy test (jl2012) +- #8854 `624a007` Fix race condition in p2p-compactblocks test (sdaftuar) +- #8857 `1f60d45` mininode: Only allow named args in wait_until (MarcoFalke) +- #8860 `0bee740` util: Move wait_bitcoinds() into stop_nodes() (MarcoFalke) +- #8882 `b73f065` Fix race conditions in p2p-compactblocks.py and sendheaders.py (sdaftuar) +- #8904 `cc6f551` Fix compact block shortids for a test case (dagurval) + +### Documentation +- #8754 `0e2c6bd` Target protobuf 2.6 in OS X build notes. (fanquake) +- #8461 `b17a3f9` Document return value of networkhashps for getmininginfo RPC endpoint (jlopp) +- #8512 `156e305` Corrected JSON typo on setban of net.cpp (sevastos) +- #8683 `8a7d7ff` Fix incorrect file name bitcoin.qrc (bitcoinsSG) +- #8891 `5e0dd9e` Update bips.md for Segregated Witness (fanquake) +- #8545 `863ae74` Update git-subtree-check.sh README (MarcoFalke) +- #8607 `486650a` Fix doxygen off-by-one comments, fix typos (MarcoFalke) +- #8560 `c493f43` Fix two VarInt examples in serialize.h (cbarcenas) +- #8737 `084cae9` UndoReadFromDisk works on undo files (rev), not on block files (paveljanik) +- #8625 `0a35573` Clarify statement about parallel jobs in rpc-tests.py (isle2983) +- #8624 `0e6d753` build: Mention curl (MarcoFalke) +- #8604 `b09e13c` build,doc: Update for 0.13.0+ and OpenBSD 5.9 (laanwj) +- #8939 `06d15fb` Update implemented bips for 0.13.1 (sipa) + +### Miscellaneous +- #8742 `d31ac72` Specify Protobuf version 2 in paymentrequest.proto (fanquake) +- #8414,#8558,#8676,#8700,#8701,#8702 Add missing copyright headers (isle2983, kazcw) +- #8899 `4ed2627` Fix wake from sleep issue with Boost 1.59.0 (fanquake) +- #8817 `bcf3806` update bitcoin-tx to output witness data (jnewbery) +- #8513 `4e5fc31` Fix a type error that would not compile on OSX. (JeremyRubin) +- #8392 `30eac2d` Fix several node initialization issues (sipa) +- #8548 `305d8ac` Use `__func__` to get function name for output printing (MarcoFalke) +- #8291 `a987431` [util] CopyrightHolders: Check for untranslated substitution (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- adlawren +- Alexey Vesnin +- Anders Øyvind Urke-Sætre +- Andrew Chow +- Anthony Towns +- BtcDrak +- Chris Stewart +- Christian Barcenas +- Christian Decker +- Cory Fields +- crowning- +- Dagur Valberg Johannsson +- David A. Harding +- Eric Lombrozo +- Ethan Heilman +- fanquake +- Gaurav Rana +- Gregory Maxwell +- instagibbs +- isle2983 +- Jameson Lopp +- Jeremy Rubin +- jnewbery +- Johnson Lau +- Jonas Schnelli +- jonnynewbs +- Justin Camarena +- Kaz Wesley +- leijurv +- Luke Dashjr +- MarcoFalke +- Marty Jones +- Matt Corallo +- Micha +- Michael Ford +- mruddy +- Pavel JanÃk +- Pieter Wuille +- rodasmith +- Sev +- Suhas Daftuar +- whythat +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 63f75fb399..61f05b0771 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -12,6 +12,7 @@ Before every minor and major release: * Update [bips.md](bips.md) to account for changes since the last release. * Update version in sources (see below) * Write release notes (see below) +* Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. Before every major release: @@ -279,6 +280,8 @@ bitcoin.org (see below for bitcoin.org update instructions). - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) - - Add release notes for the new version to the directory `doc/release-notes` in git master + - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) + + - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. - Celebrate diff --git a/doc/tor.md b/doc/tor.md index 2f376af4c7..a05979fca8 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -99,10 +99,10 @@ This means that if Tor is running (and proper authentication has been configured Bitcoin Core automatically creates a hidden service to listen on. This will positively affect the number of available .onion nodes. -This new feature is enabled by default if Bitcoin Core is listening, and -a connection to Tor can be made. It can be configured with the `-listenonion`, -`-torcontrol` and `-torpassword` settings. To show verbose debugging -information, pass `-debug=tor`. +This new feature is enabled by default if Bitcoin Core is listening (`-listen`), and +requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0` +and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings. +To show verbose debugging information, pass `-debug=tor`. Connecting to Tor's control socket API requires one of two authentication methods to be configured. For cookie authentication the user running bitcoind must have write access diff --git a/doc/unit-tests.md b/doc/unit-tests.md deleted file mode 100644 index afaece829c..0000000000 --- a/doc/unit-tests.md +++ /dev/null @@ -1,18 +0,0 @@ -Compiling/running unit tests ------------------------------------- - -Unit tests will be automatically compiled if dependencies were met in `./configure` -and tests weren't explicitly disabled. - -After configuring, they can be run with `make check`. - -To run the bitcoind tests manually, launch `src/test/test_bitcoin`. - -To add more bitcoind tests, add `BOOST_AUTO_TEST_CASE` functions to the existing -.cpp files in the `test/` directory or add new .cpp files that -implement new BOOST_AUTO_TEST_SUITE sections. - -To run the bitcoin-qt tests manually, launch `src/qt/test/test_bitcoin-qt` - -To add more bitcoin-qt tests, add them to the `src/qt/test/` directory and -the `src/qt/test/test_main.cpp` file. diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index fd7f32b5c6..2858de645d 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -1,6 +1,6 @@ """ - Copyright 2011 Jeff Garzik + Copyright (c) 2011 Jeff Garzik AuthServiceProxy has the following improvements over python-jsonrpc's ServiceProxy class: diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 6a34d64cd5..5ca6d9d015 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.7.0</string> + <string>10.8.0</string> <key>LSArchitecturePriority</key> <array> diff --git a/share/rpcuser/rpcuser.py b/share/rpcuser/rpcuser.py index 9fd176908b..f806a810e0 100755 --- a/share/rpcuser/rpcuser.py +++ b/share/rpcuser/rpcuser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +# Copyright (c) 2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/Makefile.am b/src/Makefile.am index e7f1d82b8b..5a5e3abcfa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -87,7 +87,6 @@ BITCOIN_CORE_H = \ checkpoints.h \ checkqueue.h \ clientversion.h \ - coincontrol.h \ coins.h \ compat.h \ compat/byteswap.h \ @@ -133,7 +132,7 @@ BITCOIN_CORE_H = \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ - support/pagelocker.h \ + support/lockedpool.h \ sync.h \ threadsafety.h \ timedata.h \ @@ -147,6 +146,7 @@ BITCOIN_CORE_H = \ utiltime.h \ validationinterface.h \ versionbits.h \ + wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ wallet/rpcwallet.h \ @@ -310,7 +310,7 @@ libbitcoin_common_a_SOURCES = \ libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ - support/pagelocker.cpp \ + support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibc_sanity.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c83432e91a..9760ad089c 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -17,7 +17,8 @@ bench_bench_bitcoin_SOURCES = \ bench/ccoins_caching.cpp \ bench/mempool_eviction.cpp \ bench/verify_script.cpp \ - bench/base58.cpp + bench/base58.cpp \ + bench/lockedpool.cpp bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5ce1bbb896..4e4cca14ca 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -50,7 +50,6 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ - test/Checkpoints_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 2e61363576..a58ad01b5a 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin developers +// Copyright (c) 2009-2014 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index ba3d620158..5cc52f6e72 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/base58.cpp b/src/base58.cpp index d1d60a6f1d..f7768b5a64 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -25,12 +25,14 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) psz++; // Skip and count leading '1's. int zeroes = 0; + int length = 0; while (*psz == '1') { zeroes++; psz++; } // Allocate enough space in big-endian base256 representation. - std::vector<unsigned char> b256(strlen(psz) * 733 / 1000 + 1); // log(58) / log(256), rounded up. + int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up. + std::vector<unsigned char> b256(size); // Process the characters. while (*psz && !isspace(*psz)) { // Decode base58 character @@ -39,12 +41,14 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) return false; // Apply "b256 = b256 * 58 + ch". int carry = ch - pszBase58; - for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); it != b256.rend(); it++) { + int i = 0; + for (std::vector<unsigned char>::reverse_iterator it = b256.rbegin(); (carry != 0 || i < length) && (it != b256.rend()); ++it, ++i) { carry += 58 * (*it); *it = carry % 256; carry /= 256; } assert(carry == 0); + length = i; psz++; } // Skip trailing spaces. @@ -53,7 +57,7 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) if (*psz != 0) return false; // Skip leading zeroes in b256. - std::vector<unsigned char>::iterator it = b256.begin(); + std::vector<unsigned char>::iterator it = b256.begin() + (size - length); while (it != b256.end() && *it == 0) it++; // Copy result into output vector. diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 1279c3e7df..a791b5b7fa 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016 the Bitcoin Core developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp new file mode 100644 index 0000000000..5df5b1ac6e --- /dev/null +++ b/src/bench/lockedpool.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "support/lockedpool.h" + +#include <iostream> +#include <vector> + +#define ASIZE 2048 +#define BITER 5000 +#define MSIZE 2048 + +static void LockedPool(benchmark::State& state) +{ + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + + std::vector<void*> addr; + for (int x=0; x<ASIZE; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + while (state.KeepRunning()) { + for (int x=0; x<BITER; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & (MSIZE-1)); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 + } + } + for (void *ptr: addr) + b.free(ptr); + addr.clear(); +} + +BENCHMARK(LockedPool); + diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 2d66448d80..8a2f380e67 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -240,7 +240,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed(strprintf("couldn't connect to server (%d %s)", response.error, http_errorstring(response.error))); + throw CConnectionFailed(strprintf("couldn't connect to server\n(make sure server is running and you are connecting to the correct RPC port: %d %s)", response.error, http_errorstring(response.error))); else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) diff --git a/src/bloom.cpp b/src/bloom.cpp index 2677652ada..d00befc61c 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -34,7 +34,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int * See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas */ isFull(false), - isEmpty(false), + isEmpty(true), nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), nTweak(nTweakIn), nFlags(nFlagsIn) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 5850016ae2..a57ab632e4 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -96,6 +96,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1479168000; // November 15th, 2016. consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1510704000; // November 15th, 2017. + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000002cb971dd56d1c583c20f90"); + /** * The message start string is designed to be unlikely to occur in normal data. * The characters are rarely used upper ASCII, not valid as UTF-8, and produce @@ -191,6 +194,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1462060800; // May 1st 2016 consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1493596800; // May 1st 2017 + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000000198b4def2baa9338d6"); + pchMessageStart[0] = 0x0b; pchMessageStart[1] = 0x11; pchMessageStart[2] = 0x09; @@ -224,6 +230,7 @@ public: fRequireStandard = false; fMineBlocksOnDemand = false; + checkpointData = (CCheckpointData) { boost::assign::map_list_of ( 546, uint256S("000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")), @@ -265,6 +272,9 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 0; consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 999999999999ULL; + // The best chain should have at least this much work. + consensus.nMinimumChainWork = uint256S("0x00"); + pchMessageStart[0] = 0xfa; pchMessageStart[1] = 0xbf; pchMessageStart[2] = 0xb5; diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index aefddce464..d22c188c16 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -55,16 +55,6 @@ namespace Checkpoints { return fWorkBefore / (fWorkBefore + fWorkAfter); } - int GetTotalBlocksEstimate(const CCheckpointData& data) - { - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - if (checkpoints.empty()) - return 0; - - return checkpoints.rbegin()->first; - } - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; diff --git a/src/checkpoints.h b/src/checkpoints.h index cd25ea5379..04346f35ff 100644 --- a/src/checkpoints.h +++ b/src/checkpoints.h @@ -19,9 +19,6 @@ struct CCheckpointData; namespace Checkpoints { -//! Return conservative estimate of total number of blocks, 0 if unknown -int GetTotalBlocksEstimate(const CCheckpointData& data); - //! Returns last CBlockIndex* in mapBlockIndex that is a checkpoint CBlockIndex* GetLastCheckpoint(const CCheckpointData& data); diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 899220bdc5..07ca535728 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin developers +// Copyright (c) 2014 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/endian.h b/src/compat/endian.h index 6bfae42c77..f7c1f9318a 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Bitcoin developers +// Copyright (c) 2014-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/params.h b/src/consensus/params.h index 0e73cace83..20efc68ade 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -61,6 +61,7 @@ struct Params { int64_t nPowTargetSpacing; int64_t nPowTargetTimespan; int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; } + uint256 nMinimumChainWork; }; } // namespace Consensus diff --git a/src/init.cpp b/src/init.cpp index d3efc9f978..31e3efb459 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -207,6 +207,7 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); + DumpMempool(); if (fFeeEstimatesInitialized) { @@ -359,13 +360,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s)")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -noconnect or -connect=0 alone to disable automatic connections")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); - strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect)")); + strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect/-noconnect)")); strUsage += HelpMessageOpt("-externalip=<ip>", _("Specify your own public address")); strUsage += HelpMessageOpt("-forcednsseed", strprintf(_("Always query for peer addresses via DNS lookup (default: %u)"), DEFAULT_FORCEDNSSEED)); - strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect)")); + strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: 1 if no -proxy or -connect/-noconnect)")); strUsage += HelpMessageOpt("-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt("-maxconnections=<n>", strprintf(_("Maintain at most <n> connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=<n>", strprintf(_("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); @@ -659,6 +660,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) LogPrintf("Stopping after block import\n"); StartShutdown(); } + + LoadMempool(); } /** Sanity checks @@ -960,8 +963,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) ::minRelayTxFee = CFeeRate(n); } - fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !Params().RequireStandard()); - if (Params().RequireStandard() && !fRequireStandard) + fRequireStandard = !GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); nBytesPerSigOp = GetArg("-bytespersigop", nBytesPerSigOp); @@ -996,7 +999,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (!mapMultiArgs["-bip9params"].empty()) { // Allow overriding BIP9 parameters for testing - if (!Params().MineBlocksOnDemand()) { + if (!chainparams.MineBlocksOnDemand()) { return InitError("BIP9 parameters may only be overridden on regtest."); } const vector<string>& deployments = mapMultiArgs["-bip9params"]; @@ -1100,6 +1103,10 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) return false; #endif // ********************************************************* Step 6: network initialization + // Note that we absolutely cannot open any actual connections + // until the very end ("start node") as the UTXO/block state + // is not yet setup and may end up being set up twice if we + // need to reindex later. assert(!g_connman); g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); @@ -1320,7 +1327,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) CleanupBlockRevFiles(); } - if (!LoadBlockIndex()) { + if (!LoadBlockIndex(chainparams)) { strLoadError = _("Error loading block database"); break; } @@ -1447,7 +1454,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (Params().GetConsensus().vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { + if (chainparams.GetConsensus().vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { // Only advertize witness capabilities if they have a reasonable start time. // This allows us to have the code merged without a defined softfork, by setting its // end time to 0. @@ -1493,13 +1500,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.NotifyBlockTip.disconnect(BlockNotifyGenesisWait); } -#ifdef ENABLE_WALLET - // Add wallet transactions that aren't already in a block to mempool - // Do this here as mempool requires genesis block to be loaded - if (pwalletMain) - pwalletMain->ReacceptWalletTransactions(); -#endif - // ********************************************************* Step 11: start node //// debug print @@ -1537,10 +1537,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.InitMessage(_("Done loading")); #ifdef ENABLE_WALLET - if (pwalletMain) { - // Run a thread to flush wallet periodically - threadGroup.create_thread(boost::bind(&ThreadFlushWalletDB, boost::ref(pwalletMain->strWalletFile))); - } + if (pwalletMain) + pwalletMain->postInitProcess(threadGroup); #endif return !fRequestShutdown; diff --git a/src/key.cpp b/src/key.cpp index aae9b042ac..b3ea98fb92 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -125,8 +125,8 @@ bool CKey::Check(const unsigned char *vch) { void CKey::MakeNewKey(bool fCompressedIn) { do { - GetStrongRandBytes(vch, sizeof(vch)); - } while (!Check(vch)); + GetStrongRandBytes(keydata.data(), keydata.size()); + } while (!Check(keydata.data())); fValid = true; fCompressed = fCompressedIn; } @@ -224,20 +224,18 @@ bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert(IsCompressed()); - unsigned char out[64]; - LockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); assert(pubkey.begin() + 33 == pubkey.end()); - BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, out); + BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(begin() + 32 == end()); - BIP32Hash(cc, nChild, 0, begin(), out); + BIP32Hash(cc, nChild, 0, begin(), vout.data()); } - memcpy(ccChild.begin(), out+32, 32); + memcpy(ccChild.begin(), vout.data()+32, 32); memcpy((unsigned char*)keyChild.begin(), begin(), 32); - bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), out); - UnlockObject(out); + bool ret = secp256k1_ec_privkey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); keyChild.fCompressed = true; keyChild.fValid = ret; return ret; @@ -253,12 +251,10 @@ bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { void CExtKey::SetMaster(const unsigned char *seed, unsigned int nSeedLen) { static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; - unsigned char out[64]; - LockObject(out); - CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(out); - key.Set(&out[0], &out[32], true); - memcpy(chaincode.begin(), &out[32], 32); - UnlockObject(out); + std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); + CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); + key.Set(&vout[0], &vout[32], true); + memcpy(chaincode.begin(), &vout[32], 32); nDepth = 0; nChild = 0; memset(vchFingerprint, 0, sizeof(vchFingerprint)); @@ -308,12 +304,10 @@ void ECC_Start() { { // Pass in a random blinding seed to the secp256k1 context. - unsigned char seed[32]; - LockObject(seed); - GetRandBytes(seed, 32); - bool ret = secp256k1_context_randomize(ctx, seed); + std::vector<unsigned char, secure_allocator<unsigned char>> vseed(32); + GetRandBytes(vseed.data(), 32); + bool ret = secp256k1_context_randomize(ctx, vseed.data()); assert(ret); - UnlockObject(seed); } secp256k1_context_sign = ctx; @@ -43,9 +43,7 @@ private: bool fCompressed; //! The actual byte data - unsigned char vch[32]; - - static_assert(sizeof(vch) == 32, "vch must be 32 bytes in length to not break serialization"); + std::vector<unsigned char, secure_allocator<unsigned char> > keydata; //! Check whether the 32-byte array pointed to be vch is valid keydata. bool static Check(const unsigned char* vch); @@ -54,37 +52,30 @@ public: //! Construct an invalid private key. CKey() : fValid(false), fCompressed(false) { - LockObject(vch); - } - - //! Copy constructor. This is necessary because of memlocking. - CKey(const CKey& secret) : fValid(secret.fValid), fCompressed(secret.fCompressed) - { - LockObject(vch); - memcpy(vch, secret.vch, sizeof(vch)); + // Important: vch must be 32 bytes in length to not break serialization + keydata.resize(32); } //! Destructor (again necessary because of memlocking). ~CKey() { - UnlockObject(vch); } friend bool operator==(const CKey& a, const CKey& b) { return a.fCompressed == b.fCompressed && a.size() == b.size() && - memcmp(&a.vch[0], &b.vch[0], a.size()) == 0; + memcmp(a.keydata.data(), b.keydata.data(), a.size()) == 0; } //! Initialize using begin and end iterators to byte data. template <typename T> void Set(const T pbegin, const T pend, bool fCompressedIn) { - if (pend - pbegin != sizeof(vch)) { + if (size_t(pend - pbegin) != keydata.size()) { fValid = false; } else if (Check(&pbegin[0])) { - memcpy(vch, (unsigned char*)&pbegin[0], sizeof(vch)); + memcpy(keydata.data(), (unsigned char*)&pbegin[0], keydata.size()); fValid = true; fCompressed = fCompressedIn; } else { @@ -93,9 +84,9 @@ public: } //! Simple read-only vector-like interface. - unsigned int size() const { return (fValid ? sizeof(vch) : 0); } - const unsigned char* begin() const { return vch; } - const unsigned char* end() const { return vch + size(); } + unsigned int size() const { return (fValid ? keydata.size() : 0); } + const unsigned char* begin() const { return keydata.data(); } + const unsigned char* end() const { return keydata.data() + size(); } //! Check whether this private key is valid. bool IsValid() const { return fValid; } diff --git a/src/main.cpp b/src/main.cpp index 50158b4687..e0c614b731 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,6 +18,7 @@ #include "init.h" #include "merkleblock.h" #include "net.h" +#include "netbase.h" #include "policy/fees.h" #include "policy/policy.h" #include "pow.h" @@ -63,7 +64,7 @@ CCriticalSection cs_main; BlockMap mapBlockIndex; CChain chainActive; CBlockIndex *pindexBestHeader = NULL; -int64_t nTimeBestReceived = 0; +int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we last received a block CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; int nScriptCheckThreads = 0; @@ -257,7 +258,7 @@ struct CBlockReject { */ struct CNodeState { //! The peer's address - CService address; + const CService address; //! Whether we have a fully established connection. bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. @@ -265,7 +266,7 @@ struct CNodeState { //! Whether this peer should be disconnected and banned (unless whitelisted). bool fShouldBan; //! String name of this peer (debugging/logging purposes). - std::string name; + const std::string name; //! List of asynchronously-determined block rejections to notify this peer about. std::vector<CBlockReject> rejects; //! The best known block we know this peer has announced. @@ -309,7 +310,7 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; - CNodeState() { + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; fShouldBan = false; @@ -354,11 +355,36 @@ void UpdatePreferredDownload(CNode* node, CNodeState* state) nPreferredDownload += state->fPreferredDownload; } -void InitializeNode(NodeId nodeid, const CNode *pnode) { - LOCK(cs_main); - CNodeState &state = mapNodeState.insert(std::make_pair(nodeid, CNodeState())).first->second; - state.name = pnode->addrName; - state.address = pnode->addr; +void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) +{ + ServiceFlags nLocalNodeServices = pnode->GetLocalServices(); + uint64_t nonce = pnode->GetLocalNonce(); + int nNodeStartingHeight = pnode->GetMyStartingHeight(); + NodeId nodeid = pnode->GetId(); + CAddress addr = pnode->addr; + + CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); + CAddress addrMe = CAddress(CService(), nLocalNodeServices); + + connman.PushMessageWithVersion(pnode, INIT_PROTO_VERSION, NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes); + + if (fLogIPs) + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + else + LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); +} + +void InitializeNode(CNode *pnode, CConnman& connman) { + CAddress addr = pnode->addr; + std::string addrName = pnode->addrName; + NodeId nodeid = pnode->GetId(); + { + LOCK(cs_main); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName))); + } + if(!pnode->fInbound) + PushNodeVersion(pnode, connman, GetTime()); } void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { @@ -501,15 +527,15 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pf if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - bool found = connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - pnodeStop->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + bool found = connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman.PushMessage(pnodeStop, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); return true; }); if(found) lNodesAnnouncingHeaderAndIDs.pop_front(); } fAnnounceUsingCMPCTBLOCK = true; - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); } } @@ -691,6 +717,16 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc CCoinsViewCache *pcoinsTip = NULL; CBlockTreeDB *pblocktree = NULL; +enum FlushStateMode { + FLUSH_STATE_NONE, + FLUSH_STATE_IF_NEEDED, + FLUSH_STATE_PERIODIC, + FLUSH_STATE_ALWAYS +}; + +// See definition for documentation +bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode); + ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -1135,7 +1171,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) { const uint256 hash = tx.GetHash(); @@ -1308,7 +1344,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); + CTxMemPoolEntry entry(tx, nFees, nAcceptTime, dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of @@ -1572,19 +1608,28 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return true; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); } + // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits + CValidationState stateDummy; + FlushStateToDisk(stateDummy, FLUSH_STATE_PERIODIC); return res; } -/** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ +bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +{ + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee); +} + +/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; @@ -1740,13 +1785,14 @@ bool IsInitialBlockDownload() return false; if (fImporting || fReindex) return true; - if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) + if (chainActive.Tip() == NULL) + return true; + if (chainActive.Tip()->nChainWork < UintToArith256(chainParams.GetConsensus().nMinimumChainWork)) + return true; + if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; - bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || - std::max(chainActive.Tip()->GetBlockTime(), pindexBestHeader->GetBlockTime()) < GetTime() - nMaxTipAge); - if (!state) - latchToFalse.store(true, std::memory_order_relaxed); - return state; + latchToFalse.store(true, std::memory_order_relaxed); + return false; } bool fLargeWorkForkFound = false; @@ -1774,7 +1820,7 @@ void CheckForkWarningConditions() { AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks - // (we assume we don't get stuck on a fork before the last checkpoint) + // (we assume we don't get stuck on a fork before finishing our initial sync) if (IsInitialBlockDownload()) return; @@ -2558,13 +2604,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return true; } -enum FlushStateMode { - FLUSH_STATE_NONE, - FLUSH_STATE_IF_NEEDED, - FLUSH_STATE_PERIODIC, - FLUSH_STATE_ALWAYS -}; - /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -2684,7 +2723,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); // New best block - nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); cvBlockChange.notify_all(); @@ -2768,10 +2806,9 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara std::vector<uint256> vHashUpdate; BOOST_FOREACH(const CTransaction &tx, block.vtx) { // ignore validation errors in resurrected transactions - list<CTransaction> removed; CValidationState stateDummy; if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { - mempool.removeRecursive(tx, removed); + mempool.removeRecursive(tx); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); } @@ -2804,7 +2841,7 @@ static int64_t nTimePostConnect = 0; * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::list<CTransaction> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>> &txChanged) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::vector<std::shared_ptr<const CTransaction>> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>> &txChanged) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2840,7 +2877,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, &txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2927,7 +2964,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::list<CTransaction>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>>& txChanged) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::vector<std::shared_ptr<const CTransaction>>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int>>& txChanged) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -3006,9 +3043,8 @@ static void NotifyHeaderTip() { CBlockIndex* pindexHeader = NULL; { LOCK(cs_main); - if (!setBlockIndexCandidates.empty()) { - pindexHeader = *setBlockIndexCandidates.rbegin(); - } + pindexHeader = pindexBestHeader; + if (pindexHeader != pindexHeaderOld) { fNotify = true; fInitialBlockDownload = IsInitialBlockDownload(); @@ -3039,7 +3075,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - std::list<CTransaction> txConflicted; + std::vector<std::shared_ptr<const CTransaction>> txConflicted; bool fInitialDownload; { LOCK(cs_main); @@ -3070,9 +3106,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, // throw all transactions though the signal-interface // while _not_ holding the cs_main lock - BOOST_FOREACH(const CTransaction &tx, txConflicted) + for(std::shared_ptr<const CTransaction> tx : txConflicted) { - GetMainSignals().SyncTransaction(tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().SyncTransaction(*tx, pindexNewTip, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); } // ... and about transactions that got confirmed: for(unsigned int i = 0; i < txChanged.size(); i++) @@ -3671,6 +3707,8 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (ppindex) *ppindex = pindex; + CheckBlockIndex(chainparams.GetConsensus()); + return true; } @@ -3698,6 +3736,11 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha // not process unrequested blocks. bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); + // TODO: Decouple this function from the block download logic by removing fRequested + // This requires some new chain datastructure to efficiently look up if a + // block is in a chain leading to a candidate for best tip, despite not + // being such a candidate itself. + // TODO: deal better with return value and error conditions for duplicate // and unrequested blocks. if (fAlreadyHave) return true; @@ -3746,13 +3789,11 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, C { { LOCK(cs_main); - bool fRequested = MarkBlockAsReceived(pblock->GetHash()); - fRequested |= fForceProcessing; // Store to disk CBlockIndex *pindex = NULL; bool fNewBlock = false; - bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp, &fNewBlock); + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fForceProcessing, dbp, &fNewBlock); if (pindex && pfrom) { mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); if (fNewBlock) pfrom->nLastBlockTime = GetTime(); @@ -3968,9 +4009,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash) return pindexNew; } -bool static LoadBlockIndexDB() +bool static LoadBlockIndexDB(const CChainParams& chainparams) { - const CChainParams& chainparams = Params(); if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; @@ -4265,6 +4305,9 @@ bool RewindBlockIndex(const CChainParams& params) return true; } +// May NOT be used after any connections are up as much +// of the peer-processing logic assumes a consistent +// block index state void UnloadBlockIndex() { LOCK(cs_main); @@ -4275,18 +4318,12 @@ void UnloadBlockIndex() mempool.clear(); mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); - nSyncStarted = 0; mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; nBlockSequenceId = 1; - mapBlockSource.clear(); - mapBlocksInFlight.clear(); - nPreferredDownload = 0; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); - mapNodeState.clear(); - recentRejects.reset(NULL); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); @@ -4299,10 +4336,10 @@ void UnloadBlockIndex() fHavePruned = false; } -bool LoadBlockIndex() +bool LoadBlockIndex(const CChainParams& chainparams) { // Load block index from databases - if (!fReindex && !LoadBlockIndexDB()) + if (!fReindex && !LoadBlockIndexDB(chainparams)) return false; return true; } @@ -4311,9 +4348,6 @@ bool InitBlockIndex(const CChainParams& chainparams) { LOCK(cs_main); - // Initialize global variables that cannot be constructed at startup. - recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); - // Check whether we're already initialized if (chainActive.Genesis() != NULL) return true; @@ -4702,6 +4736,11 @@ std::string GetWarnings(const std::string& strFor) // blockchain -> download logic notification // +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { + // Initialize global variables that cannot be constructed at startup. + recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); +} + void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; connman->SetBestHeight(nNewHeight); @@ -4728,6 +4767,8 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } }); } + + nTimeBestReceived = GetTime(); } void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { @@ -4885,9 +4926,9 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) - pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + connman.PushMessageWithFlag(pfrom, SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); else if (inv.type == MSG_WITNESS_BLOCK) - pfrom->PushMessage(NetMsgType::BLOCK, block); + connman.PushMessage(pfrom, NetMsgType::BLOCK, block); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -4900,7 +4941,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } if (sendMerkleBlock) { - pfrom->PushMessage(NetMsgType::MERKLEBLOCK, merkleBlock); + connman.PushMessage(pfrom, NetMsgType::MERKLEBLOCK, merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -4909,7 +4950,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, block.vtx[pair.first]); + connman.PushMessageWithFlag(pfrom, SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, block.vtx[pair.first]); } // else // no response @@ -4923,9 +4964,9 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - pfrom->PushMessageWithFlag(fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + connman.PushMessageWithFlag(pfrom, fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); } else - pfrom->PushMessageWithFlag(fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + connman.PushMessageWithFlag(pfrom, fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -4936,7 +4977,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // wait for other stuff first. vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - pfrom->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pfrom, NetMsgType::INV, vInv); pfrom->hashContinue.SetNull(); } } @@ -4947,14 +4988,14 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool push = false; auto mi = mapRelay.find(inv.hash); if (mi != mapRelay.end()) { - pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); + connman.PushMessageWithFlag(pfrom, inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); push = true; } else if (pfrom->timeLastMempoolReq) { auto txinfo = mempool.info(inv.hash); // To protect privacy, do not answer getdata using the mempool when // that TX couldn't have been INVed in reply to a MEMPOOL request. if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); + connman.PushMessageWithFlag(pfrom, inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); push = true; } } @@ -4981,7 +5022,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // do that because they want to know about (and store and rebroadcast and // risk analyze) the dependencies of transactions relevant to them, without // having to download the entire memory pool. - pfrom->PushMessage(NetMsgType::NOTFOUND, vNotFound); + connman.PushMessage(pfrom, NetMsgType::NOTFOUND, vNotFound); } } @@ -5007,8 +5048,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (!(pfrom->GetLocalServices() & NODE_BLOOM) && (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD || - strCommand == NetMsgType::FILTERCLEAR)) + strCommand == NetMsgType::FILTERADD)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); @@ -5032,7 +5072,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Each connection can only send one version message if (pfrom->nVersion != 0) { - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + connman.PushMessageWithVersion(pfrom, INIT_PROTO_VERSION, NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; @@ -5052,7 +5092,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pfrom->nServicesExpected & ~pfrom->nServices) { LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + connman.PushMessageWithVersion(pfrom, INIT_PROTO_VERSION, NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); pfrom->fDisconnect = true; return false; @@ -5062,7 +5102,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, + connman.PushMessageWithVersion(pfrom, INIT_PROTO_VERSION, NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION)); pfrom->fDisconnect = true; return false; @@ -5103,7 +5143,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Be shy and don't send version until we hear if (pfrom->fInbound) - pfrom->PushVersion(); + PushNodeVersion(pfrom, connman, GetAdjustedTime()); pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); @@ -5120,8 +5160,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } // Change version - pfrom->PushMessage(NetMsgType::VERACK); - pfrom->ssSend.SetVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); + connman.PushMessageWithVersion(pfrom, INIT_PROTO_VERSION, NetMsgType::VERACK); + pfrom->SetSendVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); if (!pfrom->fInbound) { @@ -5144,7 +5184,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Get recent addresses if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || connman.GetAddressCount() < 1000) { - pfrom->PushMessage(NetMsgType::GETADDR); + connman.PushMessage(pfrom, NetMsgType::GETADDR); pfrom->fGetAddr = true; } connman.MarkAddressGood(pfrom->addr); @@ -5191,7 +5231,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - pfrom->PushMessage(NetMsgType::SENDHEADERS); + connman.PushMessage(pfrom, NetMsgType::SENDHEADERS); } if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -5202,9 +5242,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom->GetLocalServices() & NODE_WITNESS) - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); nCMPCTBLOCKVersion = 1; - pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + connman.PushMessage(pfrom, NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); } } @@ -5332,7 +5372,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // time the block arrives, the header chain leading up to it is already validated. Not // doing this will result in the received block being rejected as an orphan in case it is // not a direct successor. - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash); CNodeState *nodestate = State(pfrom->GetId()); if (CanDirectFetch(chainparams.GetConsensus()) && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER && @@ -5368,7 +5408,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } if (!vToFetch.empty()) - pfrom->PushMessage(NetMsgType::GETDATA, vToFetch); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vToFetch); } @@ -5443,6 +5483,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, BlockTransactionsRequest req; vRecv >> req; + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); @@ -5466,7 +5508,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } resp.txn[i] = block.vtx[req.indexes[i]]; } - pfrom->PushMessageWithFlag(State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCKTXN, resp); + connman.PushMessageWithFlag(pfrom, State(pfrom->GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCKTXN, resp); } @@ -5515,7 +5557,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // headers message). In both cases it's safe to update // pindexBestHeaderSent to be our tip. nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); - pfrom->PushMessage(NetMsgType::HEADERS, vHeaders); + connman.PushMessage(pfrom, NetMsgType::HEADERS, vHeaders); } @@ -5678,13 +5720,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->id, FormatStateMessage(state)); if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); } } - FlushStateToDisk(state, FLUSH_STATE_PERIODIC); } @@ -5698,7 +5739,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!IsInitialBlockDownload()) - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); return true; } @@ -5731,7 +5772,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vInv); } return true; } @@ -5775,7 +5816,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vInv); return true; } @@ -5799,7 +5840,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman); } else { req.blockhash = pindex->GetBlockHash(); - pfrom->PushMessage(NetMsgType::GETBLOCKTXN, req); + connman.PushMessage(pfrom, NetMsgType::GETBLOCKTXN, req); } } } else { @@ -5808,7 +5849,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); - pfrom->PushMessage(NetMsgType::GETDATA, vInv); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vInv); return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -5820,8 +5861,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman); } } - - CheckBlockIndex(chainparams.GetConsensus()); } else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing @@ -5829,35 +5868,44 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, BlockTransactions resp; vRecv >> resp; - LOCK(cs_main); + CBlock block; + bool fBlockRead = false; + { + LOCK(cs_main); - map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); - return true; - } + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; - CBlock block; - ReadStatus status = partialBlock.FillBlock(block, resp.txn); - if (status == READ_STATUS_INVALID) { - MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist - Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); - return true; - } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); - pfrom->PushMessage(NetMsgType::GETDATA, invs); - } else { + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + connman.PushMessage(pfrom, NetMsgType::GETDATA, invs); + } else { + MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer + fBlockRead = true; + } + } // Don't hold cs_main when we call into ProcessNewBlock + if (fBlockRead) { CValidationState state; - ProcessNewBlock(state, chainparams, pfrom, &block, false, NULL); + // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, + // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) + ProcessNewBlock(state, chainparams, pfrom, &block, true, NULL); int nDoS; if (state.IsInvalid(nDoS)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); if (nDoS > 0) { LOCK(cs_main); @@ -5905,7 +5953,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // nUnconnectingHeaders gets reset back to 0. if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -5952,7 +6000,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); - pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256()); + connman.PushMessage(pfrom, NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256()); } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); @@ -6005,12 +6053,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - pfrom->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pfrom, NetMsgType::GETDATA, vGetData); } } } - - CheckBlockIndex(chainparams.GetConsensus()); } NotifyHeaderTip(); @@ -6029,11 +6075,17 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Such an unrequested block may still be processed, subject to the // conditions in AcceptBlock(). bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + { + LOCK(cs_main); + // Also always process if we requested the block explicitly, as we may + // need it even though it is not a candidate for a new best tip. + forceProcessing |= MarkBlockAsReceived(block.GetHash()); + } ProcessNewBlock(state, chainparams, pfrom, &block, forceProcessing, NULL); int nDoS; if (state.IsInvalid(nDoS)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes - pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + connman.PushMessage(pfrom, NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); if (nDoS > 0) { LOCK(cs_main); @@ -6110,7 +6162,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - pfrom->PushMessage(NetMsgType::PONG, nonce); + connman.PushMessage(pfrom, NetMsgType::PONG, nonce); } } @@ -6222,8 +6274,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (strCommand == NetMsgType::FILTERCLEAR) { LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); + if (pfrom->GetLocalServices() & NODE_BLOOM) { + delete pfrom->pfilter; + pfrom->pfilter = new CBloomFilter(); + } pfrom->fRelayTxes = true; } @@ -6364,7 +6418,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman) } catch (const std::ios_base::failure& e) { - pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message")); + connman.PushMessageWithVersion(pfrom, INIT_PROTO_VERSION, NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message")); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv @@ -6453,11 +6507,11 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - pto->PushMessage(NetMsgType::PING, nonce); + connman.PushMessage(pto, NetMsgType::PING, nonce); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - pto->PushMessage(NetMsgType::PING); + connman.PushMessage(pto, NetMsgType::PING); } } @@ -6488,14 +6542,14 @@ bool SendMessages(CNode* pto, CConnman& connman) // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - pto->PushMessage(NetMsgType::ADDR, vAddr); + connman.PushMessage(pto, NetMsgType::ADDR, vAddr); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - pto->PushMessage(NetMsgType::ADDR, vAddr); + connman.PushMessage(pto, NetMsgType::ADDR, vAddr); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); @@ -6518,7 +6572,7 @@ bool SendMessages(CNode* pto, CConnman& connman) } BOOST_FOREACH(const CBlockReject& reject, state.rejects) - pto->PushMessage(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock); + connman.PushMessage(pto, NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock); state.rejects.clear(); // Start block sync @@ -6541,7 +6595,7 @@ bool SendMessages(CNode* pto, CConnman& connman) if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); - pto->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256()); + connman.PushMessage(pto, NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256()); } } @@ -6630,7 +6684,7 @@ bool SendMessages(CNode* pto, CConnman& connman) CBlock block; assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); - pto->PushMessageWithFlag(state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + connman.PushMessageWithFlag(pto, state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { @@ -6642,7 +6696,7 @@ bool SendMessages(CNode* pto, CConnman& connman) LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); } - pto->PushMessage(NetMsgType::HEADERS, vHeaders); + connman.PushMessage(pto, NetMsgType::HEADERS, vHeaders); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -6688,7 +6742,7 @@ bool SendMessages(CNode* pto, CConnman& connman) BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, NetMsgType::INV, vInv); vInv.clear(); } } @@ -6734,7 +6788,7 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->filterInventoryKnown.insert(hash); vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, NetMsgType::INV, vInv); vInv.clear(); } } @@ -6800,7 +6854,7 @@ bool SendMessages(CNode* pto, CConnman& connman) } } if (vInv.size() == MAX_INV_SZ) { - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, NetMsgType::INV, vInv); vInv.clear(); } pto->filterInventoryKnown.insert(hash); @@ -6808,7 +6862,7 @@ bool SendMessages(CNode* pto, CConnman& connman) } } if (!vInv.empty()) - pto->PushMessage(NetMsgType::INV, vInv); + connman.PushMessage(pto, NetMsgType::INV, vInv); // Detect whether we're stalling nNow = GetTimeMicros(); @@ -6869,7 +6923,7 @@ bool SendMessages(CNode* pto, CConnman& connman) vGetData.push_back(inv); if (vGetData.size() >= 1000) { - pto->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pto, NetMsgType::GETDATA, vGetData); vGetData.clear(); } } else { @@ -6879,7 +6933,7 @@ bool SendMessages(CNode* pto, CConnman& connman) pto->mapAskFor.erase(pto->mapAskFor.begin()); } if (!vGetData.empty()) - pto->PushMessage(NetMsgType::GETDATA, vGetData); + connman.PushMessage(pto, NetMsgType::GETDATA, vGetData); // // Message: feefilter @@ -6892,7 +6946,7 @@ bool SendMessages(CNode* pto, CConnman& connman) if (timeNow > pto->nextSendTimeFeeFilter) { CAmount filterToSend = filterRounder.round(currentFilter); if (filterToSend != pto->lastSentFeeFilter) { - pto->PushMessage(NetMsgType::FEEFILTER, filterToSend); + connman.PushMessage(pto, NetMsgType::FEEFILTER, filterToSend); pto->lastSentFeeFilter = filterToSend; } pto->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); @@ -6924,6 +6978,119 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); } +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(void) +{ + int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "r"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t skipped = 0; + int64_t failed = 0; + int64_t nNow = GetTime(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + double prioritydummy = 0; + while (num--) { + CTransaction tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + mempool.PrioritiseTransaction(tx.GetHash(), tx.GetHash().ToString(), prioritydummy, amountdelta); + } + CValidationState state; + if (nTime + nExpiryTimeout > nNow) { + LOCK(cs_main); + AcceptToMemoryPoolWithTime(mempool, state, tx, true, NULL, nTime); + if (state.IsValid()) { + ++count; + } else { + ++failed; + } + } else { + ++skipped; + } + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + return true; +} + +void DumpMempool(void) +{ + int64_t start = GetTimeMicros(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + + { + LOCK(mempool.cs); + for (const auto &i : mempool.mapDeltas) { + mapDeltas[i.first] = i.second.first; + } + vinfo = mempool.infoAll(); + } + + int64_t mid = GetTimeMicros(); + + try { + FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "w"); + if (!filestr) { + return; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << (int64_t)i.nTime; + file << (int64_t)i.nFeeDelta; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + FileCommit(file.Get()); + file.fclose(); + RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + int64_t last = GetTimeMicros(); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*0.000001, (last-mid)*0.000001); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + } +} + class CMainCleanup { public: diff --git a/src/main.h b/src/main.h index 3eab9b89da..e80314a64b 100644 --- a/src/main.h +++ b/src/main.h @@ -237,7 +237,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB /** Initialize a new block tree database + block data on disk */ bool InitBlockIndex(const CChainParams& chainparams); /** Load the block tree and coins database from disk */ -bool LoadBlockIndex(); +bool LoadBlockIndex(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ @@ -291,6 +291,10 @@ void PruneAndFlush(); bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); +/** (try to) add transaction to memory pool with a specified acceptance time **/ +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); + /** Convert CValidationState to a human-readable message for logging */ std::string FormatStateMessage(const CValidationState &state); @@ -529,6 +533,12 @@ static const unsigned int REJECT_ALREADY_KNOWN = 0x101; /** Transaction conflicts with a transaction already known */ static const unsigned int REJECT_CONFLICT = 0x102; +/** Dump the mempool to disk. */ +void DumpMempool(); + +/** Load the mempool from disk. */ +bool LoadMempool(); + // The following things handle network-processing logic // (and should be moved to a separate file) @@ -542,7 +552,7 @@ private: CConnman* connman; public: - PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) {} + PeerLogicValidation(CConnman* connmanIn); virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); virtual void BlockChecked(const CBlock& block, const CValidationState& state); diff --git a/src/memusage.h b/src/memusage.h index 3810bfad07..2e3c5a9b92 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin developers +// Copyright (c) 2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net.cpp b/src/net.cpp index 1bca168d1d..4ab8ef98ab 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -64,6 +64,7 @@ const static std::string NET_MESSAGE_COMMAND_OTHER = "*other*"; static const uint64_t RANDOMIZER_ID_NETGROUP = 0x6c0edd8036ef4036ULL; // SHA256("netgroup")[0:8] +static const uint64_t RANDOMIZER_ID_LOCALHOSTNONCE = 0xd93e69e2bbfa5735ULL; // SHA256("localhostnonce")[0:8] // // Global state variables // @@ -389,18 +390,18 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addrman.Attempt(addrConnect, fCountFailure); // Add node - CNode* pnode = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), pszDest ? pszDest : "", false); - GetNodeSignals().InitializeNode(pnode->GetId(), pnode); + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, pszDest ? pszDest : "", false); + pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); + pnode->nTimeConnected = GetTime(); pnode->AddRef(); - + GetNodeSignals().InitializeNode(pnode, *this); { LOCK(cs_vNodes); vNodes.push_back(pnode); } - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); - pnode->nTimeConnected = GetTime(); - return pnode; } else if (!proxyConnectionFailed) { // If connecting to the node failed, and failure is not caused by a problem connecting to @@ -446,23 +447,6 @@ void CNode::CloseSocketDisconnect() vRecvMsg.clear(); } -void CNode::PushVersion() -{ - int64_t nTime = (fInbound ? GetAdjustedTime() : GetTime()); - CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService(), addr.nServices)); - CAddress addrMe = CAddress(CService(), nLocalServices); - if (fLogIPs) - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nMyStartingHeight, addrMe.ToString(), addrYou.ToString(), id); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nMyStartingHeight, addrMe.ToString(), id); - PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalServices, nTime, addrYou, addrMe, - nLocalHostNonce, strSubVersion, nMyStartingHeight, ::fRelayTxes); -} - - - - - void CConnman::ClearBanned() { { @@ -825,7 +809,7 @@ struct NodeEvictionCandidate int64_t nMinPingUsecTime; int64_t nLastBlockTime; int64_t nLastTXTime; - bool fNetworkNode; + bool fRelevantServices; bool fRelayTxes; bool fBloomFilter; CAddress addr; @@ -850,7 +834,7 @@ static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvict { // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. if (a.nLastBlockTime != b.nLastBlockTime) return a.nLastBlockTime < b.nLastBlockTime; - if (a.fNetworkNode != b.fNetworkNode) return b.fNetworkNode; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; return a.nTimeConnected > b.nTimeConnected; } @@ -885,7 +869,8 @@ bool CConnman::AttemptToEvictConnection() if (node->fDisconnect) continue; NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, - node->nLastBlockTime, node->nLastTXTime, node->fNetworkNode, + node->nLastBlockTime, node->nLastTXTime, + (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } @@ -970,7 +955,6 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { CAddress addr; int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - assert(nMaxInbound > 0); if (hSocket != INVALID_SOCKET) if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) @@ -1025,10 +1009,13 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } } - CNode* pnode = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), "", true); - GetNodeSignals().InitializeNode(pnode->GetId(), pnode); + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; + GetNodeSignals().InitializeNode(pnode, *this); LogPrint("net", "connection from %s accepted\n", addr.ToString()); @@ -1053,7 +1040,7 @@ void CConnman::ThreadSocketHandler() BOOST_FOREACH(CNode* pnode, vNodesCopy) { if (pnode->fDisconnect || - (pnode->GetRefCount() <= 0 && pnode->vRecvMsg.empty() && pnode->nSendSize == 0 && pnode->ssSend.empty())) + (pnode->GetRefCount() <= 0 && pnode->vRecvMsg.empty() && pnode->nSendSize == 0)) { // remove from vNodes vNodes.erase(remove(vNodes.begin(), vNodes.end(), pnode), vNodes.end()); @@ -1157,10 +1144,6 @@ void CConnman::ThreadSocketHandler() { TRY_LOCK(pnode->cs_vSend, lockSend); if (lockSend) { - if (pnode->nOptimisticBytesWritten) { - RecordBytesSent(pnode->nOptimisticBytesWritten); - pnode->nOptimisticBytesWritten = 0; - } if (!pnode->vSendMsg.empty()) { FD_SET(pnode->hSocket, &fdsetSend); continue; @@ -2119,8 +2102,12 @@ bool CConnman::Start(boost::thread_group& threadGroup, CScheduler& scheduler, st if (pnodeLocalHost == NULL) { CNetAddr local; LookupHost("127.0.0.1", local, false); - pnodeLocalHost = new CNode(GetNewNodeId(), nLocalServices, GetBestHeight(), INVALID_SOCKET, CAddress(CService(local, 0), nLocalServices), 0); - GetNodeSignals().InitializeNode(pnodeLocalHost->GetId(), pnodeLocalHost); + + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + + pnodeLocalHost = new CNode(id, nLocalServices, GetBestHeight(), INVALID_SOCKET, CAddress(CService(local, 0), nLocalServices), 0, nonce); + GetNodeSignals().InitializeNode(pnodeLocalHost, *this); } // @@ -2138,8 +2125,9 @@ bool CConnman::Start(boost::thread_group& threadGroup, CScheduler& scheduler, st // Initiate outbound connections from -addnode threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "addcon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenAddedConnections, this)))); - // Initiate outbound connections - threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); + // Initiate outbound connections unless connect=0 + if (!mapArgs.count("-connect") || mapMultiArgs["-connect"].size() != 1 || mapMultiArgs["-connect"][0] != "0") + threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "opencon", boost::function<void()>(boost::bind(&CConnman::ThreadOpenConnections, this)))); // Process messages threadGroup.create_thread(boost::bind(&TraceThread<boost::function<void()> >, "msghand", boost::function<void()>(boost::bind(&CConnman::ThreadMessageHandler, this)))); @@ -2471,50 +2459,20 @@ int CConnman::GetBestHeight() const return nBestHeight.load(std::memory_order_acquire); } -void CNode::Fuzz(int nChance) -{ - if (!fSuccessfullyConnected) return; // Don't fuzz initial handshake - if (GetRand(nChance) != 0) return; // Fuzz 1 of every nChance messages - - switch (GetRand(3)) - { - case 0: - // xor a random byte with a random value: - if (!ssSend.empty()) { - CDataStream::size_type pos = GetRand(ssSend.size()); - ssSend[pos] ^= (unsigned char)(GetRand(256)); - } - break; - case 1: - // delete a random byte: - if (!ssSend.empty()) { - CDataStream::size_type pos = GetRand(ssSend.size()); - ssSend.erase(ssSend.begin()+pos); - } - break; - case 2: - // insert a random byte at a random position - { - CDataStream::size_type pos = GetRand(ssSend.size()); - char ch = (char)GetRand(256); - ssSend.insert(ssSend.begin()+pos, ch); - } - break; - } - // Chance of more than one change half the time: - // (more changes exponentially less likely): - Fuzz(2); -} - unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, const std::string& addrNameIn, bool fInboundIn) : - ssSend(SER_NETWORK, INIT_PROTO_VERSION), +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string& addrNameIn, bool fInboundIn) : addr(addrIn), + fInbound(fInboundIn), + id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), - filterInventoryKnown(50000, 0.000001) + filterInventoryKnown(50000, 0.000001), + nLocalHostNonce(nLocalHostNonceIn), + nLocalServices(nLocalServicesIn), + nMyStartingHeight(nMyStartingHeightIn), + nSendVersion(0) { nServices = NODE_NONE; nServicesExpected = NODE_NONE; @@ -2533,7 +2491,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn fOneShot = false; fClient = false; // set by version message fFeeler = false; - fInbound = fInboundIn; fNetworkNode = false; fSuccessfullyConnected = false; fDisconnect = false; @@ -2562,12 +2519,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn minFeeFilter = 0; lastSentFeeFilter = 0; nextSendTimeFeeFilter = 0; - id = idIn; - nOptimisticBytesWritten = 0; - nLocalServices = nLocalServicesIn; - - GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); - nMyStartingHeight = nMyStartingHeightIn; BOOST_FOREACH(const std::string &msg, getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; @@ -2577,10 +2528,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn LogPrint("net", "Added connection to %s peer=%d\n", addrName, id); else LogPrint("net", "Added connection peer=%d\n", id); - - // Be shy and don't send version until we hear - if (hSocket != INVALID_SOCKET && !fInbound) - PushVersion(); } CNode::~CNode() @@ -2625,65 +2572,50 @@ void CNode::AskFor(const CInv& inv) mapAskFor.insert(std::make_pair(nRequestTime, inv)); } -void CNode::BeginMessage(const char* pszCommand) EXCLUSIVE_LOCK_FUNCTION(cs_vSend) +CDataStream CConnman::BeginMessage(CNode* pnode, int nVersion, int flags, const std::string& sCommand) { - ENTER_CRITICAL_SECTION(cs_vSend); - assert(ssSend.size() == 0); - ssSend << CMessageHeader(Params().MessageStart(), pszCommand, 0); - LogPrint("net", "sending: %s ", SanitizeString(pszCommand)); + return {SER_NETWORK, (nVersion ? nVersion : pnode->GetSendVersion()) | flags, CMessageHeader(Params().MessageStart(), sCommand.c_str(), 0) }; } -void CNode::AbortMessage() UNLOCK_FUNCTION(cs_vSend) +void CConnman::EndMessage(CDataStream& strm) { - ssSend.clear(); - - LEAVE_CRITICAL_SECTION(cs_vSend); + // Set the size + assert(strm.size () >= CMessageHeader::HEADER_SIZE); + unsigned int nSize = strm.size() - CMessageHeader::HEADER_SIZE; + WriteLE32((uint8_t*)&strm[CMessageHeader::MESSAGE_SIZE_OFFSET], nSize); + // Set the checksum + uint256 hash = Hash(strm.begin() + CMessageHeader::HEADER_SIZE, strm.end()); + memcpy((char*)&strm[CMessageHeader::CHECKSUM_OFFSET], hash.begin(), CMessageHeader::CHECKSUM_SIZE); - LogPrint("net", "(aborted)\n"); } -void CNode::EndMessage(const char* pszCommand) UNLOCK_FUNCTION(cs_vSend) +void CConnman::PushMessage(CNode* pnode, CDataStream& strm, const std::string& sCommand) { - // The -*messagestest options are intentionally not documented in the help message, - // since they are only used during development to debug the networking code and are - // not intended for end-users. - if (mapArgs.count("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 2)) == 0) - { - LogPrint("net", "dropmessages DROPPING SEND MESSAGE\n"); - AbortMessage(); - return; - } - if (mapArgs.count("-fuzzmessagestest")) - Fuzz(GetArg("-fuzzmessagestest", 10)); - - if (ssSend.size() == 0) - { - LEAVE_CRITICAL_SECTION(cs_vSend); + if(strm.empty()) return; - } - // Set the size - unsigned int nSize = ssSend.size() - CMessageHeader::HEADER_SIZE; - WriteLE32((uint8_t*)&ssSend[CMessageHeader::MESSAGE_SIZE_OFFSET], nSize); - - //log total amount of bytes per command - mapSendBytesPerMsgCmd[std::string(pszCommand)] += nSize + CMessageHeader::HEADER_SIZE; - // Set the checksum - uint256 hash = Hash(ssSend.begin() + CMessageHeader::HEADER_SIZE, ssSend.end()); - assert(ssSend.size () >= CMessageHeader::CHECKSUM_OFFSET + CMessageHeader::CHECKSUM_SIZE); - memcpy((char*)&ssSend[CMessageHeader::CHECKSUM_OFFSET], hash.begin(), CMessageHeader::CHECKSUM_SIZE); - - LogPrint("net", "(%d bytes) peer=%d\n", nSize, id); + unsigned int nSize = strm.size() - CMessageHeader::HEADER_SIZE; + LogPrint("net", "sending %s (%d bytes) peer=%d\n", SanitizeString(sCommand.c_str()), nSize, pnode->id); - std::deque<CSerializeData>::iterator it = vSendMsg.insert(vSendMsg.end(), CSerializeData()); - ssSend.GetAndClear(*it); - nSendSize += (*it).size(); + size_t nBytesSent = 0; + { + LOCK(pnode->cs_vSend); + if(pnode->hSocket == INVALID_SOCKET) { + return; + } + bool optimisticSend(pnode->vSendMsg.empty()); + pnode->vSendMsg.emplace_back(strm.begin(), strm.end()); - // If write queue empty, attempt "optimistic write" - if (it == vSendMsg.begin()) - nOptimisticBytesWritten += SocketSendData(this); + //log total amount of bytes per command + pnode->mapSendBytesPerMsgCmd[sCommand] += strm.size(); + pnode->nSendSize += strm.size(); - LEAVE_CRITICAL_SECTION(cs_vSend); + // If write queue empty, attempt "optimistic write" + if (optimisticSend == true) + nBytesSent = SocketSendData(pnode); + } + if (nBytesSent) + RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -136,6 +136,33 @@ public: bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); + template <typename... Args> + void PushMessageWithVersionAndFlag(CNode* pnode, int nVersion, int flag, const std::string& sCommand, Args&&... args) + { + auto msg(BeginMessage(pnode, nVersion, flag, sCommand)); + ::SerializeMany(msg, msg.nType, msg.nVersion, std::forward<Args>(args)...); + EndMessage(msg); + PushMessage(pnode, msg, sCommand); + } + + template <typename... Args> + void PushMessageWithFlag(CNode* pnode, int flag, const std::string& sCommand, Args&&... args) + { + PushMessageWithVersionAndFlag(pnode, 0, flag, sCommand, std::forward<Args>(args)...); + } + + template <typename... Args> + void PushMessageWithVersion(CNode* pnode, int nVersion, const std::string& sCommand, Args&&... args) + { + PushMessageWithVersionAndFlag(pnode, nVersion, 0, sCommand, std::forward<Args>(args)...); + } + + template <typename... Args> + void PushMessage(CNode* pnode, const std::string& sCommand, Args&&... args) + { + PushMessageWithVersionAndFlag(pnode, 0, 0, sCommand, std::forward<Args>(args)...); + } + template<typename Callable> bool ForEachNodeContinueIf(Callable&& func) { @@ -345,6 +372,10 @@ private: unsigned int GetReceiveFloodSize() const; + CDataStream BeginMessage(CNode* node, int nVersion, int flags, const std::string& sCommand); + void PushMessage(CNode* pnode, CDataStream& strm, const std::string& sCommand); + void EndMessage(CDataStream& strm); + // Network stats void RecordBytesRecv(uint64_t bytes); void RecordBytesSent(uint64_t bytes); @@ -428,7 +459,7 @@ struct CNodeSignals { boost::signals2::signal<bool (CNode*, CConnman&), CombinerAll> ProcessMessages; boost::signals2::signal<bool (CNode*, CConnman&), CombinerAll> SendMessages; - boost::signals2::signal<void (NodeId, const CNode*)> InitializeNode; + boost::signals2::signal<void (CNode*, CConnman&)> InitializeNode; boost::signals2::signal<void (NodeId, bool&)> FinalizeNode; }; @@ -553,15 +584,14 @@ public: /** Information about a peer */ class CNode { + friend class CConnman; public: // socket ServiceFlags nServices; ServiceFlags nServicesExpected; SOCKET hSocket; - CDataStream ssSend; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent - uint64_t nOptimisticBytesWritten; uint64_t nSendBytes; std::deque<CSerializeData> vSendMsg; CCriticalSection cs_vSend; @@ -589,7 +619,7 @@ public: bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; bool fClient; - bool fInbound; + const bool fInbound; bool fNetworkNode; bool fSuccessfullyConnected; bool fDisconnect; @@ -603,7 +633,7 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; int nRefCount; - NodeId id; + const NodeId id; const uint64_t nKeyedNetGroup; protected: @@ -611,9 +641,6 @@ protected: mapMsgCmdSize mapSendBytesPerMsgCmd; mapMsgCmdSize mapRecvBytesPerMsgCmd; - // Basic fuzz-testing - void Fuzz(int nChance); // modifies ssSend - public: uint256 hashContinue; int nStartingHeight; @@ -669,7 +696,7 @@ public: CAmount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, const std::string &addrNameIn = "", bool fInboundIn = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: @@ -677,10 +704,11 @@ private: void operator=(const CNode&); - uint64_t nLocalHostNonce; + const uint64_t nLocalHostNonce; // Services offered to this peer - ServiceFlags nLocalServices; - int nMyStartingHeight; + const ServiceFlags nLocalServices; + const int nMyStartingHeight; + int nSendVersion; public: NodeId GetId() const { @@ -691,6 +719,10 @@ public: return nLocalHostNonce; } + int GetMyStartingHeight() const { + return nMyStartingHeight; + } + int GetRefCount() { assert(nRefCount >= 0); @@ -716,6 +748,25 @@ public: BOOST_FOREACH(CNetMessage &msg, vRecvMsg) msg.SetVersion(nVersionIn); } + void SetSendVersion(int nVersionIn) + { + // Send version may only be changed in the version message, and + // only one version message is allowed per session. We can therefore + // treat this value as const and even atomic as long as it's only used + // once the handshake is complete. Any attempt to set this twice is an + // error. + assert(nSendVersion == 0); + nSendVersion = nVersionIn; + } + + int GetSendVersion() const + { + // The send version should always be explicitly set to + // INIT_PROTO_VERSION rather than using this value until the handshake + // is complete. See PushMessageWithVersion(). + assert(nSendVersion != 0); + return nSendVersion; + } CNode* AddRef() { @@ -778,193 +829,6 @@ public: void AskFor(const CInv& inv); - // TODO: Document the postcondition of this function. Is cs_vSend locked? - void BeginMessage(const char* pszCommand) EXCLUSIVE_LOCK_FUNCTION(cs_vSend); - - // TODO: Document the precondition of this function. Is cs_vSend locked? - void AbortMessage() UNLOCK_FUNCTION(cs_vSend); - - // TODO: Document the precondition of this function. Is cs_vSend locked? - void EndMessage(const char* pszCommand) UNLOCK_FUNCTION(cs_vSend); - - void PushVersion(); - - - void PushMessage(const char* pszCommand) - { - try - { - BeginMessage(pszCommand); - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1> - void PushMessage(const char* pszCommand, const T1& a1) - { - try - { - BeginMessage(pszCommand); - ssSend << a1; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - /** Send a message containing a1, serialized with flag flag. */ - template<typename T1> - void PushMessageWithFlag(int flag, const char* pszCommand, const T1& a1) - { - try - { - BeginMessage(pszCommand); - WithOrVersion(&ssSend, flag) << a1; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7, const T8& a8) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7 << a8; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - - template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9> - void PushMessage(const char* pszCommand, const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5, const T6& a6, const T7& a7, const T8& a8, const T9& a9) - { - try - { - BeginMessage(pszCommand); - ssSend << a1 << a2 << a3 << a4 << a5 << a6 << a7 << a8 << a9; - EndMessage(pszCommand); - } - catch (...) - { - AbortMessage(); - throw; - } - } - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index c07cd2eff8..7113390cdf 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,10 +14,9 @@ #include "util.h" void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay, std::string _dataTypeString) + unsigned int maxConfirms, double _decay) { decay = _decay; - dataTypeString = _dataTypeString; for (unsigned int i = 0; i < defaultBuckets.size(); i++) { buckets.push_back(defaultBuckets[i]); bucketMap[defaultBuckets[i]] = i; @@ -87,10 +86,10 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, int maxbucketindex = buckets.size() - 1; - // requireGreater means we are looking for the lowest fee/priority such that all higher - // values pass, so we start at maxbucketindex (highest fee) and look at successively + // requireGreater means we are looking for the lowest feerate such that all higher + // values pass, so we start at maxbucketindex (highest feerate) and look at successively // smaller buckets until we reach failure. Otherwise, we are looking for the highest - // fee/priority such that all lower values fail, and we go in the opposite direction. + // feerate such that all lower values fail, and we go in the opposite direction. unsigned int startbucket = requireGreater ? maxbucketindex : 0; int step = requireGreater ? -1 : 1; @@ -107,7 +106,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); - // Start counting from highest(default) or lowest fee/pri transactions + // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { curFarBucket = bucket; nConf += confAvg[confTarget - 1][bucket]; @@ -145,8 +144,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double median = -1; double txSum = 0; - // Calculate the "average" fee of the best bucket range that met success conditions - // Find the bucket with the median transaction and then report the average fee from that bucket + // Calculate the "average" feerate of the best bucket range that met success conditions + // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; @@ -166,8 +165,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, } } - LogPrint("estimatefee", "%3d: For conf success %s %4.2f need %s %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, dataTypeString, + LogPrint("estimatefee", "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", + confTarget, requireGreater ? ">" : "<", successBreakPoint, requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); @@ -200,10 +199,10 @@ void TxConfirmStats::Read(CAutoFile& filein) filein >> fileBuckets; numBuckets = fileBuckets.size(); if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 fee/pri buckets"); + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); filein >> fileAvg; if (fileAvg.size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri average bucket count"); + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); filein >> fileTxCtAvg; if (fileTxCtAvg.size() != numBuckets) throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); @@ -213,9 +212,9 @@ void TxConfirmStats::Read(CAutoFile& filein) throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); for (unsigned int i = 0; i < maxConfirms; i++) { if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in fee/pri conf average bucket count"); + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire fee estimate data file and not + // Now that we've processed the entire feerate estimate data file and not // thrown any errors, we can copy it to our data structures decay = fileDecay; buckets = fileBuckets; @@ -242,8 +241,8 @@ void TxConfirmStats::Read(CAutoFile& filein) for (unsigned int i = 0; i < buckets.size(); i++) bucketMap[buckets[i]] = i; - LogPrint("estimatefee", "Reading estimates: %u %s buckets counting confirms up to %u blocks\n", - numBuckets, dataTypeString, maxConfirms); + LogPrint("estimatefee", "Reading estimates: %u buckets counting confirms up to %u blocks\n", + numBuckets, maxConfirms); } unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) @@ -251,7 +250,6 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) unsigned int bucketindex = bucketMap.lower_bound(val)->second; unsigned int blockIndex = nBlockHeight % unconfTxs.size(); unconfTxs[blockIndex][bucketindex]++; - LogPrint("estimatefee", "adding to %s", dataTypeString); return bucketindex; } @@ -291,12 +289,10 @@ void CBlockPolicyEstimator::removeTx(uint256 hash) hash.ToString().c_str()); return; } - TxConfirmStats *stats = pos->second.stats; unsigned int entryHeight = pos->second.blockHeight; unsigned int bucketIndex = pos->second.bucketIndex; - if (stats != NULL) - stats->removeTx(entryHeight, nBestSeenHeight, bucketIndex); + feeStats.removeTx(entryHeight, nBestSeenHeight, bucketIndex); mapMemPoolTxs.erase(hash); } @@ -309,45 +305,14 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate& _minRelayFee) vfeelist.push_back(bucketBoundary); } vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "FeeRate"); - - minTrackedPriority = AllowFreeThreshold() < MIN_PRIORITY ? MIN_PRIORITY : AllowFreeThreshold(); - std::vector<double> vprilist; - for (double bucketBoundary = minTrackedPriority; bucketBoundary <= MAX_PRIORITY; bucketBoundary *= PRI_SPACING) { - vprilist.push_back(bucketBoundary); - } - vprilist.push_back(INF_PRIORITY); - priStats.Initialize(vprilist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, "Priority"); - - feeUnlikely = CFeeRate(0); - feeLikely = CFeeRate(INF_FEERATE); - priUnlikely = 0; - priLikely = INF_PRIORITY; -} - -bool CBlockPolicyEstimator::isFeeDataPoint(const CFeeRate &fee, double pri) -{ - if ((pri < minTrackedPriority && fee >= minTrackedFee) || - (pri < priUnlikely && fee > feeLikely)) { - return true; - } - return false; -} - -bool CBlockPolicyEstimator::isPriDataPoint(const CFeeRate &fee, double pri) -{ - if ((fee < minTrackedFee && pri >= minTrackedPriority) || - (fee < feeUnlikely && pri > priLikely)) { - return true; - } - return false; + feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool fCurrentEstimate) { unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); - if (mapMemPoolTxs[hash].stats != NULL) { + if (mapMemPoolTxs.count(hash)) { LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); return; @@ -371,30 +336,11 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo return; } - // Fees are stored and reported as BTC-per-kb: + // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); - // Want the priority of the tx at confirmation. However we don't know - // what that will be and its too hard to continue updating it - // so use starting priority as a proxy - double curPri = entry.GetPriority(txHeight); mapMemPoolTxs[hash].blockHeight = txHeight; - - LogPrint("estimatefee", "Blockpolicy mempool tx %s ", hash.ToString().substr(0,10)); - // Record this as a priority estimate - if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { - mapMemPoolTxs[hash].stats = &priStats; - mapMemPoolTxs[hash].bucketIndex = priStats.NewTx(txHeight, curPri); - } - // Record this as a fee estimate - else if (isFeeDataPoint(feeRate, curPri)) { - mapMemPoolTxs[hash].stats = &feeStats; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); - } - else { - LogPrint("estimatefee", "not adding"); - } - LogPrint("estimatefee", "\n"); + mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); } void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry& entry) @@ -417,21 +363,10 @@ void CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM return; } - // Fees are stored and reported as BTC-per-kb: + // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); - // Want the priority of the tx at confirmation. The priority when it - // entered the mempool could easily be very small and change quickly - double curPri = entry.GetPriority(nBlockHeight); - - // Record this as a priority estimate - if (entry.GetFee() == 0 || isPriDataPoint(feeRate, curPri)) { - priStats.Record(blocksToConfirm, curPri); - } - // Record this as a fee estimate - else if (isFeeDataPoint(feeRate, curPri)) { - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); - } + feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, @@ -452,41 +387,15 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, if (!fCurrentEstimate) return; - // Update the dynamic cutoffs - // a fee/priority is "likely" the reason your tx was included in a block if >85% of such tx's - // were confirmed in 2 blocks and is "unlikely" if <50% were confirmed in 10 blocks - LogPrint("estimatefee", "Blockpolicy recalculating dynamic cutoffs:\n"); - priLikely = priStats.EstimateMedianVal(2, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBlockHeight); - if (priLikely == -1) - priLikely = INF_PRIORITY; - - double feeLikelyEst = feeStats.EstimateMedianVal(2, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBlockHeight); - if (feeLikelyEst == -1) - feeLikely = CFeeRate(INF_FEERATE); - else - feeLikely = CFeeRate(feeLikelyEst); - - priUnlikely = priStats.EstimateMedianVal(10, SUFFICIENT_PRITXS, UNLIKELY_PCT, false, nBlockHeight); - if (priUnlikely == -1) - priUnlikely = 0; - - double feeUnlikelyEst = feeStats.EstimateMedianVal(10, SUFFICIENT_FEETXS, UNLIKELY_PCT, false, nBlockHeight); - if (feeUnlikelyEst == -1) - feeUnlikely = CFeeRate(0); - else - feeUnlikely = CFeeRate(feeUnlikelyEst); - - // Clear the current block states + // Clear the current block state feeStats.ClearCurrent(nBlockHeight); - priStats.ClearCurrent(nBlockHeight); // Repopulate the current block states for (unsigned int i = 0; i < entries.size(); i++) processBlockTx(nBlockHeight, entries[i]); - // Update all exponential averages with the current block states + // Update all exponential averages with the current block state feeStats.UpdateMovingAverages(); - priStats.UpdateMovingAverages(); LogPrint("estimatefee", "Blockpolicy after updating estimates for %u confirmed entries, new mempool map size %u\n", entries.size(), mapMemPoolTxs.size()); @@ -522,7 +431,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun if (answerFoundAtTarget) *answerFoundAtTarget = confTarget - 1; - // If mempool is limiting txs , return at least the min fee from the mempool + // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); if (minPoolFee > 0 && minPoolFee > median) return CFeeRate(minPoolFee); @@ -535,51 +444,38 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun double CBlockPolicyEstimator::estimatePriority(int confTarget) { - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) - return -1; - - return priStats.EstimateMedianVal(confTarget, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + return -1; } double CBlockPolicyEstimator::estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > priStats.GetMaxConfirms()) - return -1; // If mempool is limiting txs, no priority txs are allowed CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); if (minPoolFee > 0) return INF_PRIORITY; - double median = -1; - while (median < 0 && (unsigned int)confTarget <= priStats.GetMaxConfirms()) { - median = priStats.EstimateMedianVal(confTarget++, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } - - if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; - - return median; + return -1; } void CBlockPolicyEstimator::Write(CAutoFile& fileout) { fileout << nBestSeenHeight; feeStats.Write(fileout); - priStats.Write(fileout); } -void CBlockPolicyEstimator::Read(CAutoFile& filein) +void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) { int nFileBestSeenHeight; filein >> nFileBestSeenHeight; feeStats.Read(filein); - priStats.Read(filein); nBestSeenHeight = nFileBestSeenHeight; + if (nFileVersion < 139900) { + TxConfirmStats priStats; + priStats.Read(filein); + } } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index 2c1ac3b934..c7d2febfa5 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICYESTIMATOR_H @@ -19,60 +19,50 @@ class CTxMemPoolEntry; class CTxMemPool; /** \class CBlockPolicyEstimator - * The BlockPolicyEstimator is used for estimating the fee or priority needed + * The BlockPolicyEstimator is used for estimating the feerate needed * for a transaction to be included in a block within a certain number of * blocks. * * At a high level the algorithm works by grouping transactions into buckets - * based on having similar priorities or fees and then tracking how long it + * based on having similar feerates and then tracking how long it * takes transactions in the various buckets to be mined. It operates under - * the assumption that in general transactions of higher fee/priority will be - * included in blocks before transactions of lower fee/priority. So for - * example if you wanted to know what fee you should put on a transaction to + * the assumption that in general transactions of higher feerate will be + * included in blocks before transactions of lower feerate. So for + * example if you wanted to know what feerate you should put on a transaction to * be included in a block within the next 5 blocks, you would start by looking - * at the bucket with the highest fee transactions and verifying that a + * at the bucket with the highest feerate transactions and verifying that a * sufficiently high percentage of them were confirmed within 5 blocks and - * then you would look at the next highest fee bucket, and so on, stopping at - * the last bucket to pass the test. The average fee of transactions in this - * bucket will give you an indication of the lowest fee you can put on a + * then you would look at the next highest feerate bucket, and so on, stopping at + * the last bucket to pass the test. The average feerate of transactions in this + * bucket will give you an indication of the lowest feerate you can put on a * transaction and still have a sufficiently high chance of being confirmed * within your desired 5 blocks. * - * When a transaction enters the mempool or is included within a block we - * decide whether it can be used as a data point for fee estimation, priority - * estimation or neither. If the value of exactly one of those properties was - * below the required minimum it can be used to estimate the other. In - * addition, if a priori our estimation code would indicate that the - * transaction would be much more quickly included in a block because of one - * of the properties compared to the other, we can also decide to use it as - * an estimate for that property. - * - * Here is a brief description of the implementation for fee estimation. - * When a transaction that counts for fee estimation enters the mempool, we + * Here is a brief description of the implementation: + * When a transaction enters the mempool, we * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of fee + * we count the number of transactions in each bucket and the total amount of feerate * paid in each bucket. Then we calculate how many blocks Y it took each * transaction to be mined and we track an array of counters in each bucket * for how long it to took transactions to get confirmed from 1 to a max of 25 * and we increment all the counters from Y up to 25. This is because for any * number Z>=Y the transaction was successfully mined within Z blocks. We * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given fee + * counter of the total number of transactions that happened in a given feerate * bucket and the total number that were confirmed in each number 1-25 blocks * or less for any bucket. We save this history by keeping an exponentially * decaying moving average of each one of these stats. Furthermore we also * keep track of the number unmined (in mempool) transactions in each bucket * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that fee bucket when calculating + * the number of transactions we've seen in that feerate bucket when calculating * an estimate for any number of confirmations below the number of blocks * they've been outstanding. */ /** - * We will instantiate two instances of this class, one to track transactions - * that were included in a block due to fee, and one for tx's included due to - * priority. We will lump transactions into a bucket according to their approximate - * fee or priority and then track how long it took for those txs to be included in a block + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block * * The tracking of unconfirmed (mempool) transactions is completely independent of the * historical tracking of transactions that have been confirmed in a block. @@ -80,7 +70,7 @@ class CTxMemPool; class TxConfirmStats { private: - //Define the buckets we will group transactions into (both fee buckets and priority buckets) + //Define the buckets we will group transactions into std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket @@ -97,16 +87,15 @@ private: // and calculate the totals for the current block to update the moving averages std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - // Sum the total priority/fee of all tx's in each bucket + // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector<double> avg; // and calculate the total for the current block to update the moving average std::vector<double> curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg fee/priority per bucket + // Combine the total value with the tx counts to calculate the avg feerate per bucket - std::string dataTypeString; double decay; // Mempool counts of outstanding transactions @@ -123,9 +112,8 @@ public: * @param defaultBuckets contains the upper limits for the bucket boundaries * @param maxConfirms max number of confirms to track * @param decay how much to decay the historical moving average per block - * @param dataTypeString for logging purposes */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay, std::string dataTypeString); + void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); /** Clear the state of the curBlock variables to start counting for the new block */ void ClearCurrent(unsigned int nBlockHeight); @@ -133,7 +121,7 @@ public: /** * Record a new transaction data point in the current block stats * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val either the fee or the priority when entered of the transaction + * @param val the feerate of the transaction * @warning blocksToConfirm is 1-based and has to be >= 1 */ void Record(int blocksToConfirm, double val); @@ -150,14 +138,14 @@ public: void UpdateMovingAverages(); /** - * Calculate a fee or priority estimate. Find the lowest value bucket (or range of buckets + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets * to make sure we have enough data points) whose transactions still have sufficient likelihood * of being confirmed within the target number of confirmations * @param confTarget target number of confirmations * @param sufficientTxVal required average number of transactions per block in a bucket range * @param minSuccess the success probability we require - * @param requireGreater return the lowest fee/pri such that all higher values pass minSuccess OR - * return the highest fee/pri such that all lower values fail minSuccess + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, @@ -184,35 +172,27 @@ static const unsigned int MAX_BLOCK_CONFIRMS = 25; /** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ static const double DEFAULT_DECAY = .998; -/** Require greater than 95% of X fee transactions to be confirmed within Y blocks for X to be big enough */ +/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ static const double MIN_SUCCESS_PCT = .95; static const double UNLIKELY_PCT = .5; -/** Require an avg of 1 tx in the combined fee bucket per block to have stat significance */ +/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ static const double SUFFICIENT_FEETXS = 1; -/** Require only an avg of 1 tx every 5 blocks in the combined pri bucket (way less pri txs) */ -static const double SUFFICIENT_PRITXS = .2; - -// Minimum and Maximum values for tracking fees and priorities +// Minimum and Maximum values for tracking feerates static const double MIN_FEERATE = 10; static const double MAX_FEERATE = 1e7; static const double INF_FEERATE = MAX_MONEY; -static const double MIN_PRIORITY = 10; -static const double MAX_PRIORITY = 1e16; static const double INF_PRIORITY = 1e9 * MAX_MONEY; -// We have to lump transactions into buckets based on fee or priority, but we want to be able -// to give accurate estimates over a large range of potential fees and priorities +// We have to lump transactions into buckets based on feerate, but we want to be able +// to give accurate estimates over a large range of potential feerates // Therefore it makes sense to exponentially space the buckets /** Spacing of FeeRate buckets */ static const double FEE_SPACING = 1.1; -/** Spacing of Priority buckets */ -static const double PRI_SPACING = 2; - /** - * We want to be able to estimate fees or priorities that are needed on tx's to be included in + * We want to be able to estimate feerates that are needed on tx's to be included in * a certain number of blocks. Every time a block is added to the best chain, this class records * stats on the transactions included in that block */ @@ -235,27 +215,26 @@ public: /** Remove a transaction from the mempool tracking stats*/ void removeTx(uint256 hash); - /** Is this transaction likely included in a block because of its fee?*/ - bool isFeeDataPoint(const CFeeRate &fee, double pri); - - /** Is this transaction likely included in a block because of its priority?*/ - bool isPriDataPoint(const CFeeRate &fee, double pri); - - /** Return a fee estimate */ + /** Return a feerate estimate */ CFeeRate estimateFee(int confTarget); - /** Estimate fee rate needed to get be included in a block within + /** Estimate feerate needed to get be included in a block within * confTarget blocks. If no answer can be given at confTarget, return an * estimate at the lowest target where one can be given. */ CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); - /** Return a priority estimate */ + /** Return a priority estimate. + * DEPRECATED + * Returns -1 + */ double estimatePriority(int confTarget); /** Estimate priority needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + * confTarget blocks. + * DEPRECATED + * Returns -1 unless mempool is currently limited then returns INF_PRIORITY + * answerFoundAtTarget is set to confTarget */ double estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); @@ -263,29 +242,23 @@ public: void Write(CAutoFile& fileout); /** Read estimation data from a file */ - void Read(CAutoFile& filein); + void Read(CAutoFile& filein, int nFileVersion); private: CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main - double minTrackedPriority; //!< Set to AllowFreeThreshold unsigned int nBestSeenHeight; struct TxStatsInfo { - TxConfirmStats *stats; unsigned int blockHeight; unsigned int bucketIndex; - TxStatsInfo() : stats(NULL), blockHeight(0), bucketIndex(0) {} + TxStatsInfo() : blockHeight(0), bucketIndex(0) {} }; // map of txids to information about that transaction std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats, priStats; - - /** Breakpoints to help determine whether a transaction was confirmed by priority or Fee */ - CFeeRate feeLikely, feeUnlikely; - double priLikely, priUnlikely; + TxConfirmStats feeStats; }; class FeeFilterRounder diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ae42b2bd74..a3eee474ab 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin developers +// Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/policy.h b/src/policy/policy.h index 814e6c0b6f..764ee27806 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin developers +// Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 133cff611d..d9b47e71bb 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 5a711dba07..139aec5760 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin developers +// Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 4afbe99fd3..7acdac17f2 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -49,11 +49,6 @@ CTxOut::CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn) scriptPubKey = scriptPubKeyIn; } -uint256 CTxOut::GetHash() const -{ - return SerializeHash(*this); -} - std::string CTxOut::ToString() const { return strprintf("CTxOut(nValue=%d.%08d, scriptPubKey=%s)", nValue / COIN, nValue % COIN, HexStr(scriptPubKey).substr(0, 30)); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 16c2e5c454..1afeb87039 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -160,8 +160,6 @@ public: return (nValue == -1); } - uint256 GetHash() const; - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const { // "Dust" is defined in terms of CTransaction::minRelayTxFee, diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index af767aa6c6..ee5102c4f9 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -717,13 +717,10 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer { if (modalOverlay) { - if (header) { - /* use clientmodels getHeaderTipHeight and getHeaderTipTime because the NotifyHeaderTip signal does not fire when updating the best header */ - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); - } - else { + if (header) + modalOverlay->setKnownBestHeight(count, blockDate); + else modalOverlay->tipUpdate(count, blockDate, nVerificationProgress); - } } if (!clientModel) return; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 86fd4ebd65..1a1671f0ee 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -13,7 +13,7 @@ #include "txmempool.h" #include "walletmodel.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "init.h" #include "main.h" // For minRelayTxFee #include "wallet/wallet.h" diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index 73223735f5..27998f90c5 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -219,7 +219,7 @@ QLabel { color: rgb(40,40,40); }</string> <item row="0" column="1"> <widget class="QLabel" name="numberOfBlocksLeft"> <property name="text"> - <string>unknown...</string> + <string>Unknown...</string> </property> </widget> </item> @@ -245,7 +245,7 @@ QLabel { color: rgb(40,40,40); }</string> </sizepolicy> </property> <property name="text"> - <string>unknown...</string> + <string>Unknown...</string> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 06e09074d1..33db9f8938 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -411,7 +411,7 @@ </property> </widget> </item> - </layout> + </layout> </item> <item> <layout class="QFormLayout" name="formLayoutCoinControl4"> @@ -1031,7 +1031,7 @@ <item> <widget class="QLabel" name="labelSmartFee3"> <property name="text"> - <string>Confirmation time:</string> + <string>Confirmation time target:</string> </property> <property name="margin"> <number>2</number> @@ -1096,6 +1096,26 @@ </widget> </item> <item> + <spacer name="horizontalSpacer_7"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QLabel" name="confirmationTargetLabel"> + <property name="text"> + <string>(count)</string> + </property> + </widget> + </item> + <item> <spacer name="horizontalSpacer_3"> <property name="orientation"> <enum>Qt::Horizontal</enum> diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index ae0d8f5f63..1a843a07ac 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -132,7 +132,8 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri if (estimateNumHeadersLeft < 24 && hasBestHeader) { ui->numberOfBlocksLeft->setText(QString::number(bestHeaderHeight - count)); } else { - ui->expectedTimeLeft->setText(tr("Unknown. Syncing Headers...")); + ui->numberOfBlocksLeft->setText(tr("Unknown. Syncing Headers (%1)...").arg(bestHeaderHeight)); + ui->expectedTimeLeft->setText(tr("Unknown...")); } } diff --git a/src/qt/paymentrequestplus.h b/src/qt/paymentrequestplus.h index a73fe5f29d..a8bfcd2ac4 100644 --- a/src/qt/paymentrequestplus.h +++ b/src/qt/paymentrequestplus.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin developers +// Copyright (c) 2011-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 4b2ba7d624..57b2179435 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -16,7 +16,7 @@ #include "walletmodel.h" #include "base58.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "main.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" @@ -110,7 +110,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); - ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); @@ -172,6 +171,13 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateMinFeeLabel(); updateSmartFeeLabel(); updateGlobalFeeVariables(); + + // set the smartfee-sliders default value (wallets default conf.target or last stored value) + QSettings settings; + if (settings.value("nSmartFeeSliderPosition").toInt() == 0) + ui->sliderSmartFee->setValue(ui->sliderSmartFee->maximum() - model->getDefaultConfirmTarget() + 1); + else + ui->sliderSmartFee->setValue(settings.value("nSmartFeeSliderPosition").toInt()); } } @@ -229,10 +235,17 @@ void SendCoinsDialog::on_sendButton_clicked() // prepare transaction for getting txFee earlier WalletModelTransaction currentTransaction(recipients); WalletModel::SendCoinsReturn prepareStatus; - if (model->getOptionsModel()->getCoinControlFeatures()) // coin control enabled - prepareStatus = model->prepareTransaction(currentTransaction, CoinControlDialog::coinControl); + + // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled + CCoinControl ctrl; + if (model->getOptionsModel()->getCoinControlFeatures()) + ctrl = *CoinControlDialog::coinControl; + if (ui->radioSmartFee->isChecked()) + ctrl.nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; else - prepareStatus = model->prepareTransaction(currentTransaction); + ctrl.nConfirmTarget = 0; + + prepareStatus = model->prepareTransaction(currentTransaction, &ctrl); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, @@ -521,7 +534,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::TransactionCommitFailed: - msgParams.first = tr("The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of wallet.dat and coins were spent in the copy but not marked as spent here."); + msgParams.first = tr("The transaction was rejected with the following reason: %1").arg(sendCoinsReturn.reasonCommitFailed); msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::AbsurdFee: @@ -576,6 +589,7 @@ void SendCoinsDialog::updateFeeSectionControls() ui->labelFeeEstimation ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeNormal ->setEnabled(ui->radioSmartFee->isChecked()); ui->labelSmartFeeFast ->setEnabled(ui->radioSmartFee->isChecked()); + ui->confirmationTargetLabel ->setEnabled(ui->radioSmartFee->isChecked()); ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); @@ -587,15 +601,17 @@ void SendCoinsDialog::updateGlobalFeeVariables() { if (ui->radioSmartFee->isChecked()) { - nTxConfirmTarget = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; payTxFee = CFeeRate(0); // set nMinimumTotalFee to 0 to not accidentally pay a custom fee CoinControlDialog::coinControl->nMinimumTotalFee = 0; + + // show the estimated reuquired time for confirmation + ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget*600)+" / "+tr("%n block(s)", "", nConfirmTarget)); } else { - nTxConfirmTarget = defaultConfirmTarget; payTxFee = CFeeRate(ui->customFee->value()); // if user has selected to set a minimum absolute fee, pass the value to coincontrol @@ -630,7 +646,7 @@ void SendCoinsDialog::updateSmartFeeLabel() if(!model || !model->getOptionsModel()) return; - int nBlocksToConfirm = defaultConfirmTarget - ui->sliderSmartFee->value(); + int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 1; int estimateFoundAtBlocks = nBlocksToConfirm; CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); if (feeRate <= CFeeRate(0)) // not enough data => minfee @@ -701,6 +717,8 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) if (!checked && model) // coin control features disabled CoinControlDialog::coinControl->SetNull(); + // make sure we set back the confirmation target + updateGlobalFeeVariables(); coinControlUpdateLabels(); } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 83dac0bd11..b0df495a98 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -26,8 +26,6 @@ QT_BEGIN_NAMESPACE class QUrl; QT_END_NAMESPACE -const int defaultConfirmTarget = 25; - /** Dialog for sending bitcoins */ class SendCoinsDialog : public QDialog { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 305cb4fefa..3490d1c1cc 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -5,6 +5,7 @@ #include "walletmodel.h" #include "addresstablemodel.h" +#include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" #include "paymentserver.h" @@ -328,8 +329,9 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran } CReserveKey *keyChange = transaction.getPossibleKeyChange(); - if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get())) - return TransactionCommitFailed; + CValidationState state; + if(!wallet->CommitTransaction(*newTx, *keyChange, g_connman.get(), state)) + return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(state.GetRejectReason())); CTransaction* t = (CTransaction*)newTx; CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -698,3 +700,8 @@ bool WalletModel::hdEnabled() const { return wallet->IsHDEnabled(); } + +int WalletModel::getDefaultConfirmTarget() const +{ + return nTxConfirmTarget; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index cdac60da36..6a5670e378 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -144,9 +144,13 @@ public: // Return status record for SendCoins, contains error id + information struct SendCoinsReturn { - SendCoinsReturn(StatusCode _status = OK): - status(_status) {} + SendCoinsReturn(StatusCode _status = OK, QString _reasonCommitFailed = "") + : status(_status), + reasonCommitFailed(_reasonCommitFailed) + { + } StatusCode status; + QString reasonCommitFailed; }; // prepare transaction for getting txfee before sending coins @@ -207,6 +211,8 @@ public: bool hdEnabled() const; + int getDefaultConfirmTarget() const; + private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f538ddcc04..8caea14adb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -151,7 +151,7 @@ UniValue getblockcount(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getblockcount\n" - "\nReturns the number of blocks in the longest block chain.\n" + "\nReturns the number of blocks in the longest blockchain.\n" "\nResult:\n" "n (numeric) The current block count\n" "\nExamples:\n" @@ -168,7 +168,7 @@ UniValue getbestblockhash(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getbestblockhash\n" - "\nReturns the hash of the best (tip) block in the longest block chain.\n" + "\nReturns the hash of the best (tip) block in the longest blockchain.\n" "\nResult\n" "\"hex\" (string) the block hash hex encoded\n" "\nExamples\n" @@ -863,7 +863,7 @@ UniValue gettxout(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" "2. n (numeric, required) vout number\n" - "3. includemempool (boolean, optional) Whether to include the mem pool\n" + "3. includemempool (boolean, optional) Whether to include the mempool\n" "\nResult:\n" "{\n" " \"bestblock\" : \"hash\", (string) the block hash\n" @@ -1027,7 +1027,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) throw runtime_error( "getblockchaininfo\n" - "Returns an object containing various state info regarding block chain processing.\n" + "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" "{\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d509dd691f..f418262f02 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -230,7 +230,7 @@ UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"errors\": \"...\" (string) Current errors\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mem pool\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" "}\n" "\nExamples:\n" @@ -810,7 +810,7 @@ UniValue estimatepriority(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw runtime_error( "estimatepriority nblocks\n" - "\nEstimates the approximate priority a zero-fee transaction needs to begin\n" + "\nDEPRECATED. Estimates the approximate priority a zero-fee transaction needs to begin\n" "confirmation within nblocks blocks.\n" "\nArguments:\n" "1. nblocks (numeric)\n" @@ -873,7 +873,7 @@ UniValue estimatesmartpriority(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw runtime_error( "estimatesmartpriority nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nDEPRECATED. WARNING: This interface is unstable and may disappear or change!\n" "\nEstimates the approximate priority a zero-fee transaction needs to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid.\n" diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index eaef4856b3..3193985803 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -57,7 +57,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"testnet\": true|false, (boolean) if the server is using testnet or not\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" @@ -450,10 +450,53 @@ UniValue setmocktime(const JSONRPCRequest& request) return NullUniValue; } +static UniValue RPCLockedMemoryInfo() +{ + LockedPool::Stats stats = LockedPoolManager::Instance().stats(); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("used", uint64_t(stats.used))); + obj.push_back(Pair("free", uint64_t(stats.free))); + obj.push_back(Pair("total", uint64_t(stats.total))); + obj.push_back(Pair("locked", uint64_t(stats.locked))); + obj.push_back(Pair("chunks_used", uint64_t(stats.chunks_used))); + obj.push_back(Pair("chunks_free", uint64_t(stats.chunks_free))); + return obj; +} + +UniValue getmemoryinfo(const JSONRPCRequest& request) +{ + /* Please, avoid using the word "pool" here in the RPC interface or help, + * as users will undoubtedly confuse it with the other "memory pool" + */ + if (request.fHelp || request.params.size() != 0) + throw runtime_error( + "getmemoryinfo\n" + "Returns an object containing information about memory usage.\n" + "\nResult:\n" + "{\n" + " \"locked\": { (json object) Information about locked memory manager\n" + " \"used\": xxxxx, (numeric) Number of bytes used\n" + " \"free\": xxxxx, (numeric) Number of bytes available in current arenas\n" + " \"total\": xxxxxxx, (numeric) Total number of bytes managed\n" + " \"locked\": xxxxxx, (numeric) Amount of bytes that succeeded locking. If this number is smaller than total, locking pages failed at some point and key data could be swapped to disk.\n" + " \"chunks_used\": xxxxx, (numeric) Number allocated chunks\n" + " \"chunks_free\": xxxxx, (numeric) Number unused chunks\n" + " }\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getmemoryinfo", "") + + HelpExampleRpc("getmemoryinfo", "") + ); + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("locked", RPCLockedMemoryInfo())); + return obj; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "control", "getinfo", &getinfo, true }, /* uses wallet if enabled */ + { "control", "getmemoryinfo", &getmemoryinfo, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, true }, { "util", "verifymessage", &verifymessage, true }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 29d0bee1b2..164e0f00e2 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -147,6 +147,8 @@ uint256 ParseHashV(const UniValue& v, string strName) strHex = v.get_str(); if (!IsHex(strHex)) // Note: IsHex("") is false throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + if (64 != strHex.length()) + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d)", strName, 64, strHex.length())); uint256 result; result.SetHex(strHex); return result; diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index b629f4278b..1d0ca0c5ac 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -69,10 +69,19 @@ struct ECCryptoClosure ECCryptoClosure instance_of_eccryptoclosure; } +/** Check that all specified flags are part of the libconsensus interface. */ +static bool verify_flags(unsigned int flags) +{ + return (flags & ~(bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL)) == 0; +} + static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptPubKeyLen, CAmount amount, const unsigned char *txTo , unsigned int txToLen, unsigned int nIn, unsigned int flags, bitcoinconsensus_error* err) { + if (!verify_flags(flags)) { + return bitcoinconsensus_ERR_INVALID_FLAGS; + } try { TxInputStream stream(SER_NETWORK, PROTOCOL_VERSION, txTo, txToLen); CTransaction tx; diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 1d2d5c23e4..1bef4fe9e9 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -42,6 +42,7 @@ typedef enum bitcoinconsensus_error_t bitcoinconsensus_ERR_TX_SIZE_MISMATCH, bitcoinconsensus_ERR_TX_DESERIALIZE, bitcoinconsensus_ERR_AMOUNT_REQUIRED, + bitcoinconsensus_ERR_INVALID_FLAGS, } bitcoinconsensus_error; /** Script verification flags */ @@ -54,6 +55,9 @@ enum bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY = (1U << 10), // enable CHECKSEQUENCEVERIFY (BIP112) bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS = (1U << 11), // enable WITNESS (BIP141) + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL = bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG | + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NULLDUMMY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY | + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY | bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS }; /// Returns 1 if the input nIn of the serialized transaction pointed to by diff --git a/src/serialize.h b/src/serialize.h index 1f51da82ff..82870c45b3 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -160,6 +160,7 @@ enum }; #define READWRITE(obj) (::SerReadWrite(s, (obj), nType, nVersion, ser_action)) +#define READWRITEMANY(...) (::SerReadWriteMany(s, nType, nVersion, ser_action, __VA_ARGS__)) /** * Implement three methods for serializable objects. These are actually wrappers over @@ -960,4 +961,52 @@ public: } }; +template<typename Stream> +void SerializeMany(Stream& s, int nType, int nVersion) +{ +} + +template<typename Stream, typename Arg> +void SerializeMany(Stream& s, int nType, int nVersion, Arg&& arg) +{ + ::Serialize(s, std::forward<Arg>(arg), nType, nVersion); +} + +template<typename Stream, typename Arg, typename... Args> +void SerializeMany(Stream& s, int nType, int nVersion, Arg&& arg, Args&&... args) +{ + ::Serialize(s, std::forward<Arg>(arg), nType, nVersion); + ::SerializeMany(s, nType, nVersion, std::forward<Args>(args)...); +} + +template<typename Stream> +inline void UnserializeMany(Stream& s, int nType, int nVersion) +{ +} + +template<typename Stream, typename Arg> +inline void UnserializeMany(Stream& s, int nType, int nVersion, Arg& arg) +{ + ::Unserialize(s, arg, nType, nVersion); +} + +template<typename Stream, typename Arg, typename... Args> +inline void UnserializeMany(Stream& s, int nType, int nVersion, Arg& arg, Args&... args) +{ + ::Unserialize(s, arg, nType, nVersion); + ::UnserializeMany(s, nType, nVersion, args...); +} + +template<typename Stream, typename... Args> +inline void SerReadWriteMany(Stream& s, int nType, int nVersion, CSerActionSerialize ser_action, Args&&... args) +{ + ::SerializeMany(s, nType, nVersion, std::forward<Args>(args)...); +} + +template<typename Stream, typename... Args> +inline void SerReadWriteMany(Stream& s, int nType, int nVersion, CSerActionUnserialize ser_action, Args&... args) +{ + ::UnserializeMany(s, nType, nVersion, args...); +} + #endif // BITCOIN_SERIALIZE_H diff --git a/src/streams.h b/src/streams.h index 7132364eb1..fa001c112a 100644 --- a/src/streams.h +++ b/src/streams.h @@ -112,6 +112,13 @@ public: Init(nTypeIn, nVersionIn); } + template <typename... Args> + CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + { + Init(nTypeIn, nVersionIn); + ::SerializeMany(*this, nType, nVersion, std::forward<Args>(args)...); + } + void Init(int nTypeIn, int nVersionIn) { nReadPos = 0; diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 1ec40fe830..67064314ef 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -6,7 +6,8 @@ #ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H -#include "support/pagelocker.h" +#include "support/lockedpool.h" +#include "support/cleanse.h" #include <string> @@ -39,20 +40,15 @@ struct secure_allocator : public std::allocator<T> { T* allocate(std::size_t n, const void* hint = 0) { - T* p; - p = std::allocator<T>::allocate(n, hint); - if (p != NULL) - LockedPageManager::Instance().LockRange(p, sizeof(T) * n); - return p; + return static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); } void deallocate(T* p, std::size_t n) { if (p != NULL) { memory_cleanse(p, sizeof(T) * n); - LockedPageManager::Instance().UnlockRange(p, sizeof(T) * n); } - std::allocator<T>::deallocate(p, n); + LockedPoolManager::Instance().free(p); } }; diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp new file mode 100644 index 0000000000..01273c9791 --- /dev/null +++ b/src/support/lockedpool.cpp @@ -0,0 +1,385 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "support/lockedpool.h" +#include "support/cleanse.h" + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#ifdef WIN32 +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0501 +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#else +#include <sys/mman.h> // for mmap +#include <sys/resource.h> // for getrlimit +#include <limits.h> // for PAGESIZE +#include <unistd.h> // for sysconf +#endif + +#include <algorithm> + +LockedPoolManager* LockedPoolManager::_instance = NULL; +std::once_flag LockedPoolManager::init_flag; + +/*******************************************************************************/ +// Utilities +// +/** Align up to power of 2 */ +static inline size_t align_up(size_t x, size_t align) +{ + return (x + align - 1) & ~(align - 1); +} + +/*******************************************************************************/ +// Implementation: Arena + +Arena::Arena(void *base_in, size_t size_in, size_t alignment_in): + base(static_cast<char*>(base_in)), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in) +{ + // Start with one free chunk that covers the entire arena + chunks_free.emplace(base, size_in); +} + +Arena::~Arena() +{ +} + +void* Arena::alloc(size_t size) +{ + // Round to next multiple of alignment + size = align_up(size, alignment); + + // Don't handle zero-sized chunks + if (size == 0) + return nullptr; + + // Pick a large enough free-chunk + auto it = std::find_if(chunks_free.begin(), chunks_free.end(), + [=](const std::map<char*, size_t>::value_type& chunk){ return chunk.second >= size; }); + if (it == chunks_free.end()) + return nullptr; + + // Create the used-chunk, taking its space from the end of the free-chunk + auto alloced = chunks_used.emplace(it->first + it->second - size, size).first; + if (!(it->second -= size)) + chunks_free.erase(it); + return reinterpret_cast<void*>(alloced->first); +} + +/* extend the Iterator if other begins at its end */ +template <class Iterator, class Pair> bool extend(Iterator it, const Pair& other) { + if (it->first + it->second == other.first) { + it->second += other.second; + return true; + } + return false; +} + +void Arena::free(void *ptr) +{ + // Freeing the NULL pointer is OK. + if (ptr == nullptr) { + return; + } + + // Remove chunk from used map + auto i = chunks_used.find(static_cast<char*>(ptr)); + if (i == chunks_used.end()) { + throw std::runtime_error("Arena: invalid or double free"); + } + auto freed = *i; + chunks_used.erase(i); + + // Add space to free map, coalescing contiguous chunks + auto next = chunks_free.upper_bound(freed.first); + auto prev = (next == chunks_free.begin()) ? chunks_free.end() : std::prev(next); + if (prev == chunks_free.end() || !extend(prev, freed)) + prev = chunks_free.emplace_hint(next, freed); + if (next != chunks_free.end() && extend(prev, *next)) + chunks_free.erase(next); +} + +Arena::Stats Arena::stats() const +{ + Arena::Stats r{ 0, 0, 0, chunks_used.size(), chunks_free.size() }; + for (const auto& chunk: chunks_used) + r.used += chunk.second; + for (const auto& chunk: chunks_free) + r.free += chunk.second; + r.total = r.used + r.free; + return r; +} + +#ifdef ARENA_DEBUG +void printchunk(char* base, size_t sz, bool used) { + std::cout << + "0x" << std::hex << std::setw(16) << std::setfill('0') << base << + " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz << + " 0x" << used << std::endl; +} +void Arena::walk() const +{ + for (const auto& chunk: chunks_used) + printchunk(chunk.first, chunk.second, true); + std::cout << std::endl; + for (const auto& chunk: chunks_free) + printchunk(chunk.first, chunk.second, false); + std::cout << std::endl; +} +#endif + +/*******************************************************************************/ +// Implementation: Win32LockedPageAllocator + +#ifdef WIN32 +/** LockedPageAllocator specialized for Windows. + */ +class Win32LockedPageAllocator: public LockedPageAllocator +{ +public: + Win32LockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +Win32LockedPageAllocator::Win32LockedPageAllocator() +{ + // Determine system page size in bytes + SYSTEM_INFO sSysInfo; + GetSystemInfo(&sSysInfo); + page_size = sSysInfo.dwPageSize; +} +void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + len = align_up(len, page_size); + void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (addr) { + // VirtualLock is used to attempt to keep keying material out of swap. Note + // that it does not provide this as a guarantee, but, in practice, memory + // that has been VirtualLock'd almost never gets written to the pagefile + // except in rare circumstances where memory is extremely low. + *lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0; + } + return addr; +} +void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + VirtualUnlock(const_cast<void*>(addr), len); +} + +size_t Win32LockedPageAllocator::GetLimit() +{ + // TODO is there a limit on windows, how to get it? + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: PosixLockedPageAllocator + +#ifndef WIN32 +/** LockedPageAllocator specialized for OSes that don't try to be + * special snowflakes. + */ +class PosixLockedPageAllocator: public LockedPageAllocator +{ +public: + PosixLockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess); + void FreeLocked(void* addr, size_t len); + size_t GetLimit(); +private: + size_t page_size; +}; + +PosixLockedPageAllocator::PosixLockedPageAllocator() +{ + // Determine system page size in bytes +#if defined(PAGESIZE) // defined in limits.h + page_size = PAGESIZE; +#else // assume some POSIX OS + page_size = sysconf(_SC_PAGESIZE); +#endif +} + +// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define +// MAP_ANON which is deprecated +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + void *addr; + len = align_up(len, page_size); + addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (addr) { + *lockingSuccess = mlock(addr, len) == 0; + } + return addr; +} +void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + munlock(addr, len); + munmap(addr, len); +} +size_t PosixLockedPageAllocator::GetLimit() +{ +#ifdef RLIMIT_MEMLOCK + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) { + if (rlim.rlim_cur != RLIM_INFINITY) { + return rlim.rlim_cur; + } + } +#endif + return std::numeric_limits<size_t>::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: LockedPool + +LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in): + allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0) +{ +} + +LockedPool::~LockedPool() +{ +} +void* LockedPool::alloc(size_t size) +{ + std::lock_guard<std::mutex> lock(mutex); + + // Don't handle impossible sizes + if (size == 0 || size > ARENA_SIZE) + return nullptr; + + // Try allocating from each current arena + for (auto &arena: arenas) { + void *addr = arena.alloc(size); + if (addr) { + return addr; + } + } + // If that fails, create a new one + if (new_arena(ARENA_SIZE, ARENA_ALIGN)) { + return arenas.back().alloc(size); + } + return nullptr; +} + +void LockedPool::free(void *ptr) +{ + std::lock_guard<std::mutex> lock(mutex); + // TODO we can do better than this linear search by keeping a map of arena + // extents to arena, and looking up the address. + for (auto &arena: arenas) { + if (arena.addressInArena(ptr)) { + arena.free(ptr); + return; + } + } + throw std::runtime_error("LockedPool: invalid address not pointing to any arena"); +} + +LockedPool::Stats LockedPool::stats() const +{ + std::lock_guard<std::mutex> lock(mutex); + LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0}; + for (const auto &arena: arenas) { + Arena::Stats i = arena.stats(); + r.used += i.used; + r.free += i.free; + r.total += i.total; + r.chunks_used += i.chunks_used; + r.chunks_free += i.chunks_free; + } + return r; +} + +bool LockedPool::new_arena(size_t size, size_t align) +{ + bool locked; + // If this is the first arena, handle this specially: Cap the upper size + // by the process limit. This makes sure that the first arena will at least + // be locked. An exception to this is if the process limit is 0: + // in this case no memory can be locked at all so we'll skip past this logic. + if (arenas.empty()) { + size_t limit = allocator->GetLimit(); + if (limit > 0) { + size = std::min(size, limit); + } + } + void *addr = allocator->AllocateLocked(size, &locked); + if (!addr) { + return false; + } + if (locked) { + cumulative_bytes_locked += size; + } else if (lf_cb) { // Call the locking-failed callback if locking failed + if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed. + allocator->FreeLocked(addr, size); + return false; + } + } + arenas.emplace_back(allocator.get(), addr, size, align); + return true; +} + +LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in): + Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in) +{ +} +LockedPool::LockedPageArena::~LockedPageArena() +{ + allocator->FreeLocked(base, size); +} + +/*******************************************************************************/ +// Implementation: LockedPoolManager +// +LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator): + LockedPool(std::move(allocator), &LockedPoolManager::LockingFailed) +{ +} + +bool LockedPoolManager::LockingFailed() +{ + // TODO: log something but how? without including util.h + return true; +} + +void LockedPoolManager::CreateInstance() +{ + // Using a local static instance guarantees that the object is initialized + // when it's first needed and also deinitialized after all objects that use + // it are done with it. I can think of one unlikely scenario where we may + // have a static deinitialization order/problem, but the check in + // LockedPoolManagerBase's destructor helps us detect if that ever happens. +#ifdef WIN32 + std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator()); +#else + std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator()); +#endif + static LockedPoolManager instance(std::move(allocator)); + LockedPoolManager::_instance = &instance; +} diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h new file mode 100644 index 0000000000..3403415436 --- /dev/null +++ b/src/support/lockedpool.h @@ -0,0 +1,231 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H +#define BITCOIN_SUPPORT_LOCKEDPOOL_H + +#include <stdint.h> +#include <list> +#include <map> +#include <mutex> +#include <memory> + +/** + * OS-dependent allocation and deallocation of locked/pinned memory pages. + * Abstract base class. + */ +class LockedPageAllocator +{ +public: + virtual ~LockedPageAllocator() {} + /** Allocate and lock memory pages. + * If len is not a multiple of the system page size, it is rounded up. + * Returns 0 in case of allocation failure. + * + * If locking the memory pages could not be accomplished it will still + * return the memory, however the lockingSuccess flag will be false. + * lockingSuccess is undefined if the allocation fails. + */ + virtual void* AllocateLocked(size_t len, bool *lockingSuccess) = 0; + + /** Unlock and free memory pages. + * Clear the memory before unlocking. + */ + virtual void FreeLocked(void* addr, size_t len) = 0; + + /** Get the total limit on the amount of memory that may be locked by this + * process, in bytes. Return size_t max if there is no limit or the limit + * is unknown. Return 0 if no memory can be locked at all. + */ + virtual size_t GetLimit() = 0; +}; + +/* An arena manages a contiguous region of memory by dividing it into + * chunks. + */ +class Arena +{ +public: + Arena(void *base, size_t size, size_t alignment); + virtual ~Arena(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t chunks_used; + size_t chunks_free; + }; + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get arena usage statistics */ + Stats stats() const; + +#ifdef ARENA_DEBUG + void walk() const; +#endif + + /** Return whether a pointer points inside this arena. + * This returns base <= ptr < (base+size) so only use it for (inclusive) + * chunk starting addresses. + */ + bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; } +private: + Arena(const Arena& other) = delete; // non construction-copyable + Arena& operator=(const Arena&) = delete; // non copyable + + /** Map of chunk address to chunk information. This class makes use of the + * sorted order to merge previous and next chunks during deallocation. + */ + std::map<char*, size_t> chunks_free; + std::map<char*, size_t> chunks_used; + /** Base address of arena */ + char* base; + /** End address of arena */ + char* end; + /** Minimum chunk alignment */ + size_t alignment; +}; + +/** Pool for locked memory chunks. + * + * To avoid sensitive key data from being swapped to disk, the memory in this pool + * is locked/pinned. + * + * An arena manages a contiguous region of memory. The pool starts out with one arena + * but can grow to multiple arenas if the need arises. + * + * Unlike a normal C heap, the administrative structures are seperate from the managed + * memory. This has been done as the sizes and bases of objects are not in themselves sensitive + * information, as to conserve precious locked memory. In some operating systems + * the amount of memory that can be locked is small. + */ +class LockedPool +{ +public: + /** Size of one arena of locked memory. This is a compromise. + * Do not set this too low, as managing many arenas will increase + * allocation and deallocation overhead. Setting it too high allocates + * more locked memory from the OS than strictly necessary. + */ + static const size_t ARENA_SIZE = 256*1024; + /** Chunk alignment. Another compromise. Setting this too high will waste + * memory, setting it too low will facilitate fragmentation. + */ + static const size_t ARENA_ALIGN = 16; + + /** Callback when allocation succeeds but locking fails. + */ + typedef bool (*LockingFailed_Callback)(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t locked; + size_t chunks_used; + size_t chunks_free; + }; + + /** Create a new LockedPool. This takes ownership of the MemoryPageLocker, + * you can only instantiate this with LockedPool(std::move(...)). + * + * The second argument is an optional callback when locking a newly allocated arena failed. + * If this callback is provided and returns false, the allocation fails (hard fail), if + * it returns true the allocation proceeds, but it could warn. + */ + LockedPool(std::unique_ptr<LockedPageAllocator> allocator, LockingFailed_Callback lf_cb_in = 0); + ~LockedPool(); + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get pool usage statistics */ + Stats stats() const; +private: + LockedPool(const LockedPool& other) = delete; // non construction-copyable + LockedPool& operator=(const LockedPool&) = delete; // non copyable + + std::unique_ptr<LockedPageAllocator> allocator; + + /** Create an arena from locked pages */ + class LockedPageArena: public Arena + { + public: + LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align); + ~LockedPageArena(); + private: + void *base; + size_t size; + LockedPageAllocator *allocator; + }; + + bool new_arena(size_t size, size_t align); + + std::list<LockedPageArena> arenas; + LockingFailed_Callback lf_cb; + size_t cumulative_bytes_locked; + /** Mutex protects access to this pool's data structures, including arenas. + */ + mutable std::mutex mutex; +}; + +/** + * Singleton class to keep track of locked (ie, non-swappable) memory, for use in + * std::allocator templates. + * + * Some implementations of the STL allocate memory in some constructors (i.e., see + * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) + * Due to the unpredictable order of static initializers, we have to make sure the + * LockedPoolManager instance exists before any other STL-based objects that use + * secure_allocator are created. So instead of having LockedPoolManager also be + * static-initialized, it is created on demand. + */ +class LockedPoolManager : public LockedPool +{ +public: + /** Return the current instance, or create it once */ + static LockedPoolManager& Instance() + { + std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + return *LockedPoolManager::_instance; + } + +private: + LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator); + + /** Create a new LockedPoolManager specialized to the OS */ + static void CreateInstance(); + /** Called when locking fails, warn the user here */ + static bool LockingFailed(); + + static LockedPoolManager* _instance; + static std::once_flag init_flag; +}; + +#endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/support/pagelocker.cpp b/src/support/pagelocker.cpp deleted file mode 100644 index 7cea2d88c5..0000000000 --- a/src/support/pagelocker.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "support/pagelocker.h" - -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#define WIN32_LEAN_AND_MEAN 1 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include <windows.h> -// This is used to attempt to keep keying material out of swap -// Note that VirtualLock does not provide this as a guarantee on Windows, -// but, in practice, memory that has been VirtualLock'd almost never gets written to -// the pagefile except in rare circumstances where memory is extremely low. -#else -#include <sys/mman.h> -#include <limits.h> // for PAGESIZE -#include <unistd.h> // for sysconf -#endif - -LockedPageManager* LockedPageManager::_instance = NULL; -boost::once_flag LockedPageManager::init_flag = BOOST_ONCE_INIT; - -/** Determine system page size in bytes */ -static inline size_t GetSystemPageSize() -{ - size_t page_size; -#if defined(WIN32) - SYSTEM_INFO sSysInfo; - GetSystemInfo(&sSysInfo); - page_size = sSysInfo.dwPageSize; -#elif defined(PAGESIZE) // defined in limits.h - page_size = PAGESIZE; -#else // assume some POSIX OS - page_size = sysconf(_SC_PAGESIZE); -#endif - return page_size; -} - -bool MemoryPageLocker::Lock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualLock(const_cast<void*>(addr), len) != 0; -#else - return mlock(addr, len) == 0; -#endif -} - -bool MemoryPageLocker::Unlock(const void* addr, size_t len) -{ -#ifdef WIN32 - return VirtualUnlock(const_cast<void*>(addr), len) != 0; -#else - return munlock(addr, len) == 0; -#endif -} - -LockedPageManager::LockedPageManager() : LockedPageManagerBase<MemoryPageLocker>(GetSystemPageSize()) -{ -} diff --git a/src/support/pagelocker.h b/src/support/pagelocker.h deleted file mode 100644 index 538bf39453..0000000000 --- a/src/support/pagelocker.h +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SUPPORT_PAGELOCKER_H -#define BITCOIN_SUPPORT_PAGELOCKER_H - -#include "support/cleanse.h" - -#include <map> - -#include <boost/thread/mutex.hpp> -#include <boost/thread/once.hpp> - -/** - * Thread-safe class to keep track of locked (ie, non-swappable) memory pages. - * - * Memory locks do not stack, that is, pages which have been locked several times by calls to mlock() - * will be unlocked by a single call to munlock(). This can result in keying material ending up in swap when - * those functions are used naively. This class simulates stacking memory locks by keeping a counter per page. - * - * @note By using a map from each page base address to lock count, this class is optimized for - * small objects that span up to a few pages, mostly smaller than a page. To support large allocations, - * something like an interval tree would be the preferred data structure. - */ -template <class Locker> -class LockedPageManagerBase -{ -public: - LockedPageManagerBase(size_t _page_size) : page_size(_page_size) - { - // Determine bitmask for extracting page from address - assert(!(_page_size & (_page_size - 1))); // size must be power of two - page_mask = ~(_page_size - 1); - } - - ~LockedPageManagerBase() - { - } - - - // For all pages in affected range, increase lock count - void LockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - if (it == histogram.end()) // Newly locked page - { - locker.Lock(reinterpret_cast<void*>(page), page_size); - histogram.insert(std::make_pair(page, 1)); - } else // Page was already locked; increase counter - { - it->second += 1; - } - } - } - - // For all pages in affected range, decrease lock count - void UnlockRange(void* p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if (!size) - return; - const size_t base_addr = reinterpret_cast<size_t>(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for (size_t page = start_page; page <= end_page; page += page_size) { - Histogram::iterator it = histogram.find(page); - assert(it != histogram.end()); // Cannot unlock an area that was not locked - // Decrease counter for page, when it is zero, the page will be unlocked - it->second -= 1; - if (it->second == 0) // Nothing on the page anymore that keeps it locked - { - // Unlock page and remove the count from histogram - locker.Unlock(reinterpret_cast<void*>(page), page_size); - histogram.erase(it); - } - } - } - - // Get number of locked pages for diagnostics - int GetLockedPageCount() - { - boost::mutex::scoped_lock lock(mutex); - return histogram.size(); - } - -private: - Locker locker; - boost::mutex mutex; - size_t page_size, page_mask; - // map of page base address to lock count - typedef std::map<size_t, int> Histogram; - Histogram histogram; -}; - - -/** - * OS-dependent memory page locking/unlocking. - * Defined as policy class to make stubbing for test possible. - */ -class MemoryPageLocker -{ -public: - /** Lock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Lock(const void* addr, size_t len); - /** Unlock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Unlock(const void* addr, size_t len); -}; - -/** - * Singleton class to keep track of locked (ie, non-swappable) memory pages, for use in - * std::allocator templates. - * - * Some implementations of the STL allocate memory in some constructors (i.e., see - * MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.) - * Due to the unpredictable order of static initializers, we have to make sure the - * LockedPageManager instance exists before any other STL-based objects that use - * secure_allocator are created. So instead of having LockedPageManager also be - * static-initialized, it is created on demand. - */ -class LockedPageManager : public LockedPageManagerBase<MemoryPageLocker> -{ -public: - static LockedPageManager& Instance() - { - boost::call_once(LockedPageManager::CreateInstance, LockedPageManager::init_flag); - return *LockedPageManager::_instance; - } - -private: - LockedPageManager(); - - static void CreateInstance() - { - // Using a local static instance guarantees that the object is initialized - // when it's first needed and also deinitialized after all objects that use - // it are done with it. I can think of one unlikely scenario where we may - // have a static deinitialization order/problem, but the check in - // LockedPageManagerBase's destructor helps us detect if that ever happens. - static LockedPageManager instance; - LockedPageManager::_instance = &instance; - } - - static LockedPageManager* _instance; - static boost::once_flag init_flag; -}; - -// -// Functions for directly locking/unlocking memory objects. -// Intended for non-dynamically allocated structures. -// -template <typename T> -void LockObject(const T& t) -{ - LockedPageManager::Instance().LockRange((void*)(&t), sizeof(T)); -} - -template <typename T> -void UnlockObject(const T& t) -{ - memory_cleanse((void*)(&t), sizeof(T)); - LockedPageManager::Instance().UnlockRange((void*)(&t), sizeof(T)); -} - -#endif // BITCOIN_SUPPORT_PAGELOCKER_H diff --git a/src/test/Checkpoints_tests.cpp b/src/test/Checkpoints_tests.cpp deleted file mode 100644 index 1b7d368e13..0000000000 --- a/src/test/Checkpoints_tests.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -// -// Unit tests for block-chain checkpoints -// - -#include "checkpoints.h" - -#include "uint256.h" -#include "test/test_bitcoin.h" -#include "chainparams.h" - -#include <boost/test/unit_test.hpp> - -using namespace std; - -BOOST_FIXTURE_TEST_SUITE(Checkpoints_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(sanity) -{ - const CCheckpointData& checkpoints = Params(CBaseChainParams::MAIN).Checkpoints(); - BOOST_CHECK(Checkpoints::GetTotalBlocksEstimate(checkpoints) >= 134444); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 97abeb7211..6eed636080 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -48,8 +48,9 @@ BOOST_AUTO_TEST_CASE(DoS_banning) { connman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, "", true); - GetNodeSignals().InitializeNode(dummyNode1.GetId(), &dummyNode1); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, "", true); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); // Should get banned SendMessages(&dummyNode1, *connman); @@ -57,8 +58,9 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, "", true); - GetNodeSignals().InitializeNode(dummyNode2.GetId(), &dummyNode2); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, "", true); + dummyNode2.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode2, *connman); dummyNode2.nVersion = 1; Misbehaving(dummyNode2.GetId(), 50); SendMessages(&dummyNode2, *connman); @@ -74,8 +76,9 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) connman->ClearBanned(); mapArgs["-banscore"] = "111"; // because 11 is my favorite number CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, "", true); - GetNodeSignals().InitializeNode(dummyNode1.GetId(), &dummyNode1); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, "", true); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); SendMessages(&dummyNode1, *connman); @@ -96,8 +99,9 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, "", true); - GetNodeSignals().InitializeNode(dummyNode.GetId(), &dummyNode); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, "", true); + dummyNode.SetSendVersion(PROTOCOL_VERSION); + GetNodeSignals().InitializeNode(&dummyNode, *connman); dummyNode.nVersion = 1; Misbehaving(dummyNode.GetId(), 100); diff --git a/src/test/README.md b/src/test/README.md index 3afdefe5fc..8f99804e10 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -1,4 +1,36 @@ -# Notes +### Compiling/running unit tests + +Unit tests will be automatically compiled if dependencies were met in `./configure` +and tests weren't explicitly disabled. + +After configuring, they can be run with `make check`. + +To run the bitcoind tests manually, launch `src/test/test_bitcoin`. + +To add more bitcoind tests, add `BOOST_AUTO_TEST_CASE` functions to the existing +.cpp files in the `test/` directory or add new .cpp files that +implement new BOOST_AUTO_TEST_SUITE sections. + +To run the bitcoin-qt tests manually, launch `src/qt/test/test_bitcoin-qt` + +To add more bitcoin-qt tests, add them to the `src/qt/test/` directory and +the `src/qt/test/test_main.cpp` file. + +### Running individual tests + +test_bitcoin has some built-in command-line arguments; for +example, to run just the getarg_tests verbosely: + + test_bitcoin --log_level=all --run_test=getarg_tests + +... or to run just the doubledash test: + + test_bitcoin --run_test=getarg_tests/doubledash + +Run `test_bitcoin --help` for the full list. + +### Note on adding test cases + The sources in this directory are unit test cases. Boost includes a unit testing framework, and since bitcoin already uses boost, it makes sense to simply use this framework rather than require developers to @@ -19,17 +51,6 @@ For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: [http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). -test_bitcoin has some built-in command-line arguments; for -example, to run just the getarg_tests verbosely: - - test_bitcoin --log_level=all --run_test=getarg_tests - -... or to run just the doubledash test: - - test_bitcoin --run_test=getarg_tests/doubledash - -Run `test_bitcoin --help` for the full list. - ### bitcoin-util-test.py The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just bitcoin-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory: @@ -37,4 +58,4 @@ The test directory also contains the bitcoin-util-test.py tool, which tests bitc ``` test/bitcoin-util-test.py --srcdir=[current directory] -```
\ No newline at end of file +``` diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 613f6c12d7..77e9df5d82 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -11,110 +11,224 @@ BOOST_FIXTURE_TEST_SUITE(allocator_tests, BasicTestingSetup) -// Dummy memory page locker for platform independent tests -static const void *last_lock_addr, *last_unlock_addr; -static size_t last_lock_len, last_unlock_len; -class TestLocker +BOOST_AUTO_TEST_CASE(arena_tests) { -public: - bool Lock(const void *addr, size_t len) + // Fake memory base address for testing + // without actually using memory. + void *synth_base = reinterpret_cast<void*>(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + void *chunk = b.alloc(1000); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(chunk != nullptr); + BOOST_CHECK(b.stats().used == 1008); // Aligned to 16 + BOOST_CHECK(b.stats().total == synth_size); // Nothing has disappeared? + b.free(chunk); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK(b.stats().free == synth_size); + try { // Test exception on double-free + b.free(chunk); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - last_lock_addr = addr; - last_lock_len = len; - return true; } - bool Unlock(const void *addr, size_t len) - { - last_unlock_addr = addr; - last_unlock_len = len; - return true; + + void *a0 = b.alloc(128); + void *a1 = b.alloc(256); + void *a2 = b.alloc(512); + BOOST_CHECK(b.stats().used == 896); + BOOST_CHECK(b.stats().total == synth_size); +#ifdef ARENA_DEBUG + b.walk(); +#endif + b.free(a0); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 768); + b.free(a1); + BOOST_CHECK(b.stats().used == 512); + void *a3 = b.alloc(128); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 640); + b.free(a2); + BOOST_CHECK(b.stats().used == 128); + b.free(a3); + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK_EQUAL(b.stats().chunks_used, 0); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + BOOST_CHECK_EQUAL(b.stats().chunks_free, 1); + + std::vector<void*> addr; + BOOST_CHECK(b.alloc(0) == nullptr); // allocating 0 always returns nullptr +#ifdef ARENA_DEBUG + b.walk(); +#endif + // Sweeping allocate all memory + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + BOOST_CHECK(b.stats().free == 0); + BOOST_CHECK(b.alloc(1024) == nullptr); // memory is full, this must return nullptr + BOOST_CHECK(b.alloc(0) == nullptr); + for (int x=0; x<1024; ++x) + b.free(addr[x]); + addr.clear(); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + + // Now in the other direction... + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + for (int x=0; x<1024; ++x) + b.free(addr[1023-x]); + addr.clear(); + + // Now allocate in smaller unequal chunks, then deallocate haphazardly + // Not all the chunks will succeed allocating, but freeing nullptr is + // allowed so that is no problem. + for (int x=0; x<2048; ++x) + addr.push_back(b.alloc(x+1)); + for (int x=0; x<2048; ++x) + b.free(addr[((x*23)%2048)^242]); + addr.clear(); + + // Go entirely wild: free and alloc interleaved, + // generate targets and sizes using pseudo-randomness. + for (int x=0; x<2048; ++x) + addr.push_back(0); + uint32_t s = 0x12345678; + for (int x=0; x<5000; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = 0; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & 2047); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 } -}; + for (void *ptr: addr) + b.free(ptr); + addr.clear(); -BOOST_AUTO_TEST_CASE(test_LockedPageManagerBase) + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); +} + +/** Mock LockedPageAllocator for testing */ +class TestLockedPageAllocator: public LockedPageAllocator { - const size_t test_page_size = 4096; - LockedPageManagerBase<TestLocker> lpm(test_page_size); - size_t addr; - last_lock_addr = last_unlock_addr = 0; - last_lock_len = last_unlock_len = 0; - - /* Try large number of small objects */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; - } - /* Try small number of page-sized objects, straddling two pages */ - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* Try small number of page-sized objects aligned to exactly one page */ - addr = test_page_size*300; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; - } - /* one very large object, straddling pages */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(600+500))); - /* one very large object, page aligned */ - lpm.LockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); - BOOST_CHECK(last_lock_addr == reinterpret_cast<void*>(test_page_size*(1200+500-1))); - - BOOST_CHECK(lpm.GetLockedPageCount() == ( - (1000*33+test_page_size-1)/test_page_size + // small objects - 101 + 100 + // page-sized objects - 501 + 500)); // large objects - BOOST_CHECK((last_lock_len & (test_page_size-1)) == 0); // always lock entire pages - BOOST_CHECK(last_unlock_len == 0); // nothing unlocked yet - - /* And unlock again */ - addr = 0; - for(int i=0; i<1000; ++i) +public: + TestLockedPageAllocator(int count_in, int lockedcount_in): count(count_in), lockedcount(lockedcount_in) {} + void* AllocateLocked(size_t len, bool *lockingSuccess) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 33); - addr += 33; + *lockingSuccess = false; + if (count > 0) { + --count; + + if (lockedcount > 0) { + --lockedcount; + *lockingSuccess = true; + } + + return reinterpret_cast<void*>(0x08000000 + (count<<24)); // Fake address, do not actually use this memory + } + return 0; } - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) + void FreeLocked(void* addr, size_t len) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; } - addr = test_page_size*300; - for(int i=0; i<100; ++i) + size_t GetLimit() { - lpm.UnlockRange(reinterpret_cast<void*>(addr), test_page_size); - addr += test_page_size; + return std::numeric_limits<size_t>::max(); } - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*600+1), test_page_size*500); - lpm.UnlockRange(reinterpret_cast<void*>(test_page_size*1200), test_page_size*500-1); +private: + int count; + int lockedcount; +}; - /* Check that everything is released */ - BOOST_CHECK(lpm.GetLockedPageCount() == 0); +BOOST_AUTO_TEST_CASE(lockedpool_tests_mock) +{ + // Test over three virtual arenas, of which one will succeed being locked + std::unique_ptr<LockedPageAllocator> x(new TestLockedPageAllocator(3, 1)); + LockedPool pool(std::move(x)); + BOOST_CHECK(pool.stats().total == 0); + BOOST_CHECK(pool.stats().locked == 0); - /* A few and unlocks of size zero (should have no effect) */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; - } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - addr = 0; - for(int i=0; i<1000; ++i) + // Ensure unreasonable requests are refused without allocating anything + void *invalid_toosmall = pool.alloc(0); + BOOST_CHECK(invalid_toosmall == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + void *invalid_toobig = pool.alloc(LockedPool::ARENA_SIZE+1); + BOOST_CHECK(invalid_toobig == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + + void *a0 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a0); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + void *a1 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a1); + void *a2 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a2); + void *a3 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a3); + void *a4 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a4); + void *a5 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a5); + // We've passed a count of three arenas, so this allocation should fail + void *a6 = pool.alloc(16); + BOOST_CHECK(!a6); + + pool.free(a0); + pool.free(a2); + pool.free(a4); + pool.free(a1); + pool.free(a3); + pool.free(a5); + BOOST_CHECK(pool.stats().total == 3*LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().used == 0); +} + +// These tests used the live LockedPoolManager object, this is also used +// by other tests so the conditions are somewhat less controllable and thus the +// tests are somewhat more error-prone. +BOOST_AUTO_TEST_CASE(lockedpool_tests_live) +{ + LockedPoolManager &pool = LockedPoolManager::Instance(); + LockedPool::Stats initial = pool.stats(); + + void *a0 = pool.alloc(16); + BOOST_CHECK(a0); + // Test reading and writing the allocated memory + *((uint32_t*)a0) = 0x1234; + BOOST_CHECK(*((uint32_t*)a0) == 0x1234); + + pool.free(a0); + try { // Test exception on double-free + pool.free(a0); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - lpm.UnlockRange(reinterpret_cast<void*>(addr), 0); - addr += 1; } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - BOOST_CHECK((last_unlock_len & (test_page_size-1)) == 0); // always unlock entire pages + // If more than one new arena was allocated for the above tests, something is wrong + BOOST_CHECK(pool.stats().total <= (initial.total + LockedPool::ARENA_SIZE)); + // Usage must be back to where it started + BOOST_CHECK(pool.stats().used == initial.used); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bctest.py b/src/test/bctest.py index d801415c70..adc5d0e418 100644 --- a/src/test/bctest.py +++ b/src/test/bctest.py @@ -1,4 +1,5 @@ -# Copyright 2014 BitPay, Inc. +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from __future__ import division,print_function,unicode_literals @@ -6,56 +7,115 @@ import subprocess import os import json import sys +import binascii +import difflib +import logging + +def parse_output(a, fmt): + """Parse the output according to specified format. + + Raise an error if the output can't be parsed.""" + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return binascii.a2b_hex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) def bctest(testDir, testObj, exeext): + """Runs a single test, comparing output and RC to expected output and RC. + + Raises an error if input can't be read, executable fails, or output/RC + are not as expected. Error is caught by bctester() and reported. + """ + # Get the exec names and arguments + execprog = testObj['exec'] + exeext + execargs = testObj['args'] + execrun = [execprog] + execargs + + # Read the input data (if there is any) + stdinCfg = None + inputData = None + if "input" in testObj: + filename = testDir + "/" + testObj['input'] + inputData = open(filename).read() + stdinCfg = subprocess.PIPE + + # Read the expected output data (if there is any) + outputFn = None + outputData = None + if "output_cmp" in testObj: + outputFn = testObj['output_cmp'] + outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) + try: + outputData = open(testDir + "/" + outputFn).read() + except: + logging.error("Output file " + outputFn + " can not be opened") + raise + if not outputData: + logging.error("Output data missing for " + outputFn) + raise Exception + + # Run the test + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) + try: + outs = proc.communicate(input=inputData) + except OSError: + logging.error("OSError, Failed to execute " + execprog) + raise + + if outputData: + # Parse command output and expected output + try: + a_parsed = parse_output(outs[0], outputType) + except Exception as e: + logging.error('Error parsing command output as %s: %s' % (outputType,e)) + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) + raise + # Compare data + if a_parsed != b_parsed: + logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + raise Exception + # Compare formatting + if outs[0] != outputData: + error_message = "Output formatting mismatch for " + outputFn + ":\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + outs[0].splitlines(True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + raise Exception + + # Compare the return code to the expected return code + wantRC = 0 + if "return_code" in testObj: + wantRC = testObj['return_code'] + if proc.returncode != wantRC: + logging.error("Return code mismatch for " + outputFn) + raise Exception + +def bctester(testDir, input_basename, buildenv): + """ Loads and parses the input file, runs all tests and reports results""" + input_filename = testDir + "/" + input_basename + raw_data = open(input_filename).read() + input_data = json.loads(raw_data) + + failed_testcases = [] - execprog = testObj['exec'] + exeext - execargs = testObj['args'] - execrun = [execprog] + execargs - stdinCfg = None - inputData = None - if "input" in testObj: - filename = testDir + "/" + testObj['input'] - inputData = open(filename).read() - stdinCfg = subprocess.PIPE - - outputFn = None - outputData = None - if "output_cmp" in testObj: - outputFn = testObj['output_cmp'] - outputData = open(testDir + "/" + outputFn).read() - if not outputData: - print("Output data missing for " + outputFn) - sys.exit(1) - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) - try: - outs = proc.communicate(input=inputData) - except OSError: - print("OSError, Failed to execute " + execprog) - sys.exit(1) - - if outputData and (outs[0] != outputData): - print("Output data mismatch for " + outputFn) - sys.exit(1) - - wantRC = 0 - if "return_code" in testObj: - wantRC = testObj['return_code'] - if proc.returncode != wantRC: - print("Return code mismatch for " + outputFn) - sys.exit(1) - -def bctester(testDir, input_basename, buildenv, verbose = False): - input_filename = testDir + "/" + input_basename - raw_data = open(input_filename).read() - input_data = json.loads(raw_data) - - for testObj in input_data: - if verbose and "description" in testObj: - print ("Testing: " + testObj["description"]) - bctest(testDir, testObj, buildenv.exeext) - if verbose and "description" in testObj: - print ("PASS") - - sys.exit(0) + for testObj in input_data: + try: + bctest(testDir, testObj, buildenv.exeext) + logging.info("PASSED: " + testObj["description"]) + except: + logging.info("FAILED: " + testObj["description"]) + failed_testcases.append(testObj["description"]) + if failed_testcases: + logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") + sys.exit(1) + else: + sys.exit(0) diff --git a/src/test/bitcoin-util-test.py b/src/test/bitcoin-util-test.py index 3099506d6d..4301b93b7c 100755 --- a/src/test/bitcoin-util-test.py +++ b/src/test/bitcoin-util-test.py @@ -1,12 +1,15 @@ #!/usr/bin/env python -# Copyright 2014 BitPay, Inc. +# Copyright 2014 BitPay Inc. +# Copyright 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from __future__ import division,print_function,unicode_literals import os +import sys import bctest import buildenv import argparse +import logging help_text="""Test framework for bitcoin utils. @@ -14,14 +17,16 @@ Runs automatically during `make check`. Can also be run manually from the src directory by specifiying the source directory: -test/bitcoin-util-test.py --src=[srcdir] +test/bitcoin-util-test.py --srcdir='srcdir' [--verbose] """ - if __name__ == '__main__': - verbose = False + # Try to get the source directory from the environment variables. This will + # be set for `make check` automated runs. If environment variable is not set, + # then get the source directory from command line args. try: srcdir = os.environ["srcdir"] + verbose = False except: parser = argparse.ArgumentParser(description=help_text) parser.add_argument('-s', '--srcdir') @@ -29,4 +34,13 @@ if __name__ == '__main__': args = parser.parse_args() srcdir = args.srcdir verbose = args.verbose - bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose) + + if verbose: + level = logging.DEBUG + else: + level = logging.ERROR + formatter = '%(asctime)s - %(levelname)s - %(message)s' + # Add the format/level to the logger + logging.basicConfig(format = formatter, level=level) + + bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv) diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 7530b013bd..b0d9184816 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -80,8 +80,8 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2].GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); - std::list<CTransaction> removed; - pool.removeRecursive(block.vtx[2], removed); + std::vector<std::shared_ptr<const CTransaction>> removed; + pool.removeRecursive(block.vtx[2], &removed); BOOST_CHECK_EQUAL(removed.size(), 1); CBlock block2; diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 033a50f94f..a73dbe725c 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -55,15 +55,15 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) CTxMemPool testPool(CFeeRate(0)); - std::list<CTransaction> removed; + std::vector<std::shared_ptr<const CTransaction>> removed; // Nothing in pool, remove should do nothing: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 1); removed.clear(); @@ -75,16 +75,16 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 2); removed.clear(); // ... make sure grandchild and child are gone: - testPool.removeRecursive(txGrandChild[0], removed); + testPool.removeRecursive(txGrandChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.removeRecursive(txChild[0], removed); + testPool.removeRecursive(txChild[0], &removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Remove parent, all children/grandchildren should go: - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 5); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.removeRecursive(txParent, removed); + testPool.removeRecursive(txParent, &removed); BOOST_CHECK_EQUAL(removed.size(), 6); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -281,12 +281,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10); // Now try removing tx10 and verify the sort order returns to normal - std::list<CTransaction> removed; - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx()); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), removed); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx()); + pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx()); /* Now check the sort on the mining score index. * Final order should be: * @@ -413,8 +412,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) /* after tx6 is mined, tx7 should move up in the sort */ std::vector<CTransaction> vtx; vtx.push_back(tx6); - std::list<CTransaction> dummy; - pool.removeForBlock(vtx, 1, dummy, false); + pool.removeForBlock(vtx, 1, NULL, false); sortedOrder.erase(sortedOrder.begin()+1); sortedOrder.pop_back(); @@ -549,12 +547,11 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(tx7.GetHash(), entry.Fee(9000LL).FromTx(tx7, &pool)); std::vector<CTransaction> vtx; - std::list<CTransaction> conflicts; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); // ... we should keep the same min fee until we get a block - pool.removeForBlock(vtx, 1, conflicts); + pool.removeForBlock(vtx, 1); SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/2); // ... then feerate should drop 1/2 each halflife diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d6d7b5716e..a94979fd77 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -137,8 +137,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, // Test that packages above the min relay fee do get included, even if one // of the transactions is below the min relay fee // Remove the low fee transaction and replace with a higher fee transaction - std::list<CTransaction> dummy; - mempool.removeRecursive(tx, dummy); + mempool.removeRecursive(tx); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); mempool.addUnchecked(hashLowFeeTx, entry.Fee(feeToUse+2).FromTx(tx)); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index f4b5768693..e0460109d5 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -164,12 +164,12 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) bool fInboundIn = false; // Test that fFeeler is false by default. - CNode* pnode1 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, pszDest, fInboundIn); + CNode* pnode1 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, pszDest, fInboundIn); BOOST_CHECK(pnode1->fInbound == false); BOOST_CHECK(pnode1->fFeeler == false); fInboundIn = true; - CNode* pnode2 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, pszDest, fInboundIn); + CNode* pnode2 = new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, pszDest, fInboundIn); BOOST_CHECK(pnode2->fInbound == true); BOOST_CHECK(pnode2->fFeeler == false); } diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 5c902387f1..38aaaba267 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -19,26 +19,18 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) CTxMemPool mpool(CFeeRate(1000)); TestMemPoolEntryHelper entry; CAmount basefee(2000); - double basepri = 1e6; CAmount deltaFee(100); - double deltaPri=5e5; - std::vector<CAmount> feeV[2]; - std::vector<double> priV[2]; + std::vector<CAmount> feeV; - // Populate vectors of increasing fees or priorities + // Populate vectors of increasing fees for (int j = 0; j < 10; j++) { - //V[0] is for fee transactions - feeV[0].push_back(basefee * (j+1)); - priV[0].push_back(0); - //V[1] is for priority transactions - feeV[1].push_back(CAmount(0)); - priV[1].push_back(basepri * pow(10, j+1)); + feeV.push_back(basefee * (j+1)); } // Store the hashes of transactions that have been - // added to the mempool by their associate fee/pri + // added to the mempool by their associate fee // txHashes[j] is populated with transactions either of - // fee = basefee * (j+1) OR pri = 10^6 * 10^(j+1) + // fee = basefee * (j+1) std::vector<uint256> txHashes[10]; // Create a transaction template @@ -46,7 +38,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (unsigned int i = 0; i < 128; i++) garbage.push_back('X'); CMutableTransaction tx; - std::list<CTransaction> dummyConflicted; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); @@ -61,19 +52,19 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // At a decay .998 and 4 fee transactions per block // This makes the tx count about 1.33 per bucket, above the 1 threshold while (blocknum < 200) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); txHashes[j].push_back(hash); } } - //Create blocks where higher fee/pri txs are included more often + //Create blocks where higher fee txs are included more often for (int h = 0; h <= blocknum%10; h++) { - // 10/10 blocks add highest fee/pri transactions + // 10/10 blocks add highest fee transactions // 9/10 blocks add 2nd highest and so on until ... - // 1/10 blocks add lowest fee/pri transactions + // 1/10 blocks add lowest fee transactions while (txHashes[9-h].size()) { std::shared_ptr<const CTransaction> ptx = mpool.get(txHashes[9-h].back()); if (ptx) @@ -81,7 +72,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[9-h].pop_back(); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); if (blocknum == 30) { // At this point we should need to combine 5 buckets to get enough data points @@ -101,7 +92,6 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } std::vector<CAmount> origFeeEst; - std::vector<double> origPriEst; // Highest feerate is 10*baseRate and gets in all blocks, // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, // third highest feerate is 8*base rate, and gets in 8/10 blocks = 80%, @@ -110,51 +100,43 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); - origPriEst.push_back(mpool.estimatePriority(i)); if (i > 1) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); - BOOST_CHECK(origPriEst[i-1] <= origPriEst[i-2]); } int mult = 11-i; BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); - BOOST_CHECK(origPriEst[i-1] < pow(10,mult) * basepri + deltaPri); - BOOST_CHECK(origPriEst[i-1] > pow(10,mult) * basepri - deltaPri); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change // We haven't decayed the moving average enough so we still have enough data points in every bucket while (blocknum < 250) - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] + deltaPri); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); } // Mine 15 more blocks with lots of transactions happening and not getting mined // Estimates should go up while (blocknum < 265) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee multiple + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); txHashes[j].push_back(hash); } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); } int answerFound; for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) == -1 || mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); - BOOST_CHECK(mpool.estimateSmartPriority(i, &answerFound) > origPriEst[answerFound-1] - deltaPri); } // Mine all those transactions @@ -167,40 +149,39 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265, dummyConflicted); + mpool.removeForBlock(block, 265); block.clear(); for (int i = 1; i < 10;i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) > origPriEst[i-1] - deltaPri); } // Mine 200 more blocks where everything is mined every block // Estimates should be below original estimates while (blocknum < 465) { - for (int j = 0; j < 10; j++) { // For each fee/pri multiple - for (int k = 0; k < 5; k++) { // add 4 fee txs for every priority tx + for (int j = 0; j < 10; j++) { // For each fee multiple + for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); + mpool.addUnchecked(hash, entry.Fee(feeV[j]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); std::shared_ptr<const CTransaction> ptx = mpool.get(hash); if (ptx) block.push_back(*ptx); + } } - mpool.removeForBlock(block, ++blocknum, dummyConflicted); + mpool.removeForBlock(block, ++blocknum); block.clear(); } for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimatePriority(i) < origPriEst[i-1] - deltaPri); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee // and that estimateSmartPriority returns essentially an infinite value - mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[0][5]).Time(GetTime()).Priority(priV[1][5]).Height(blocknum).FromTx(tx, &mpool)); - // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[0][5] + mpool.addUnchecked(tx.GetHash(), entry.Fee(feeV[5]).Time(GetTime()).Priority(0).Height(blocknum).FromTx(tx, &mpool)); + // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5] mpool.TrimToSize(1); - BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[0][5]); + BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 561adb8ea2..3169afb13a 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -173,11 +173,14 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << tx2; - if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect, message); - } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, flags, NULL) == expect,message); + int libconsensus_flags = flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL; + if (libconsensus_flags == flags) { + if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + } else { + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(begin_ptr(scriptPubKey), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(begin_ptr(scriptPubKey), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, NULL) == expect,message); + } } #endif } diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index bec2c7459d..4c0fdc77f7 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -10,11 +10,54 @@ #include <stdint.h> #include <boost/test/unit_test.hpp> - using namespace std; BOOST_FIXTURE_TEST_SUITE(serialize_tests, BasicTestingSetup) +class CSerializeMethodsTestSingle +{ +protected: + int intval; + bool boolval; + std::string stringval; + const char* charstrval; + CTransaction txval; +public: + CSerializeMethodsTestSingle() = default; + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(txvalin){} + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(intval); + READWRITE(boolval); + READWRITE(stringval); + READWRITE(FLATDATA(charstrval)); + READWRITE(txval); + } + + bool operator==(const CSerializeMethodsTestSingle& rhs) + { + return intval == rhs.intval && \ + boolval == rhs.boolval && \ + stringval == rhs.stringval && \ + strcmp(charstrval, rhs.charstrval) == 0 && \ + txval == rhs.txval; + } +}; + +class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle +{ +public: + using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITEMANY(intval, boolval, stringval, FLATDATA(charstrval), txval); + } +}; + BOOST_AUTO_TEST_CASE(sizes) { BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0), 0)); @@ -297,4 +340,30 @@ BOOST_AUTO_TEST_CASE(insert_delete) BOOST_CHECK_EQUAL(ss.size(), 0); } +BOOST_AUTO_TEST_CASE(class_methods) +{ + int intval(100); + bool boolval(true); + std::string stringval("testing"); + const char* charstrval("testing charstr"); + CMutableTransaction txval; + CSerializeMethodsTestSingle methodtest1(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestSingle methodtest3; + CSerializeMethodsTestMany methodtest4; + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + BOOST_CHECK(methodtest1 == methodtest2); + ss << methodtest1; + ss >> methodtest4; + ss << methodtest2; + ss >> methodtest3; + BOOST_CHECK(methodtest1 == methodtest2); + BOOST_CHECK(methodtest2 == methodtest3); + BOOST_CHECK(methodtest3 == methodtest4); + + CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, FLATDATA(charstrval), txval); + ss2 >> methodtest3; + BOOST_CHECK(methodtest3 == methodtest4); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/univalue_tests.cpp b/src/test/univalue_tests.cpp index 45d480c816..7f794fcbe9 100644 --- a/src/test/univalue_tests.cpp +++ b/src/test/univalue_tests.cpp @@ -1,4 +1,4 @@ -// Copyright 2014 BitPay, Inc. +// Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0f1c166abc..45135a5f73 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -503,7 +503,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransaction>& removed) +void CTxMemPool::removeRecursive(const CTransaction &origTx, std::vector<std::shared_ptr<const CTransaction>>* removed) { // Remove transaction from memory pool { @@ -530,8 +530,10 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list<CTransact BOOST_FOREACH(txiter it, txToRemove) { CalculateDescendants(it, setAllRemoves); } - BOOST_FOREACH(txiter it, setAllRemoves) { - removed.push_back(it->GetTx()); + if (removed) { + BOOST_FOREACH(txiter it, setAllRemoves) { + removed->emplace_back(it->GetSharedTx()); + } } RemoveStaged(setAllRemoves, false); } @@ -541,7 +543,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions LOCK(cs); - list<CTransaction> transactionsToRemove; + setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction& tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); @@ -549,16 +551,16 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem if (!CheckFinalTx(tx, flags) || !CheckSequenceLocks(tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be invalid // So it's critical that we remove the tx and not depend on the LockPoints. - transactionsToRemove.push_back(tx); + txToRemove.insert(it); } else if (it->GetSpendsCoinbase()) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); + if (nCheckFrequency != 0) assert(coins); if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { - transactionsToRemove.push_back(tx); + txToRemove.insert(it); break; } } @@ -567,13 +569,14 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem mapTx.modify(it, update_lock_points(lp)); } } - BOOST_FOREACH(const CTransaction& tx, transactionsToRemove) { - list<CTransaction> removed; - removeRecursive(tx, removed); + setEntries setAllRemoves; + for (txiter it : txToRemove) { + CalculateDescendants(it, setAllRemoves); } + RemoveStaged(setAllRemoves, false); } -void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed) +void CTxMemPool::removeConflicts(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); @@ -594,7 +597,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> * Called when a block is connected. Removes from mempool and updates the miner fee estimator. */ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate) + std::vector<std::shared_ptr<const CTransaction>>* conflicts, bool fCurrentEstimate) { LOCK(cs); std::vector<CTxMemPoolEntry> entries; @@ -830,6 +833,10 @@ void CTxMemPool::queryHashes(vector<uint256>& vtxid) } } +static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { + return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize()), it->GetModifiedFee() - it->GetFee()}; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -838,7 +845,7 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const std::vector<TxMempoolInfo> ret; ret.reserve(mapTx.size()); for (auto it : iters) { - ret.push_back(TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize())}); + ret.push_back(GetInfo(it)); } return ret; @@ -859,7 +866,7 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) return TxMempoolInfo(); - return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; + return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const @@ -888,7 +895,7 @@ CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const { try { LOCK(cs); - fileout << 109900; // version required to read: 0.10.99 or later + fileout << 139900; // version required to read: 0.13.99 or later fileout << CLIENT_VERSION; // version that wrote the file minerPolicyEstimator->Write(fileout); } @@ -907,9 +914,8 @@ CTxMemPool::ReadFeeEstimates(CAutoFile& filein) filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein); + minerPolicyEstimator->Read(filein, nVersionThatWrote); } catch (const std::exception&) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); diff --git a/src/txmempool.h b/src/txmempool.h index 1763930ba0..9b0ca4655e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -6,9 +6,12 @@ #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H -#include <list> #include <memory> #include <set> +#include <map> +#include <vector> +#include <utility> +#include <string> #include "amount.h" #include "coins.h" @@ -326,6 +329,9 @@ struct TxMempoolInfo /** Feerate of the transaction. */ CFeeRate feeRate; + + /** The fee delta. */ + int64_t nFeeDelta; }; /** @@ -521,11 +527,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); - void removeRecursive(const CTransaction &tx, std::list<CTransaction>& removed); + void removeRecursive(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed = NULL); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); - void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); + void removeConflicts(const CTransaction &tx, std::vector<std::shared_ptr<const CTransaction>>* removed = NULL); void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, - std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); + std::vector<std::shared_ptr<const CTransaction>>* conflicts = NULL, bool fCurrentEstimate = true); void clear(); void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); diff --git a/src/coincontrol.h b/src/wallet/coincontrol.h index e33adc4d2b..08d23688ff 100644 --- a/src/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_COINCONTROL_H -#define BITCOIN_COINCONTROL_H +#ifndef BITCOIN_WALLET_COINCONTROL_H +#define BITCOIN_WALLET_COINCONTROL_H #include "primitives/transaction.h" @@ -22,6 +22,8 @@ public: bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true CFeeRate nFeeRate; + //! Override the default confirmation target, 0 = use default + int nConfirmTarget; CCoinControl() { @@ -37,6 +39,7 @@ public: nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; + nConfirmTarget = 0; } bool HasSelected() const @@ -73,4 +76,4 @@ private: std::set<COutPoint> setSelected; }; -#endif // BITCOIN_COINCONTROL_H +#endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 190f8ecf2a..31ee060677 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -48,12 +48,12 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v int i = 0; if (nDerivationMethod == 0) - i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, chKey, chIV); + i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, vchKey.data(), vchIV.data()); if (i != (int)WALLET_CRYPTO_KEY_SIZE) { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); return false; } @@ -66,8 +66,8 @@ bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector<unsigne if (chNewKey.size() != WALLET_CRYPTO_KEY_SIZE || chNewIV.size() != WALLET_CRYPTO_IV_SIZE) return false; - memcpy(&chKey[0], &chNewKey[0], sizeof chKey); - memcpy(&chIV[0], &chNewIV[0], sizeof chIV); + memcpy(vchKey.data(), chNewKey.data(), chNewKey.size()); + memcpy(vchIV.data(), chNewIV.data(), chNewIV.size()); fKeySet = true; return true; @@ -82,7 +82,7 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned // n + AES_BLOCKSIZE bytes vchCiphertext.resize(vchPlaintext.size() + AES_BLOCKSIZE); - AES256CBCEncrypt enc(chKey, chIV, true); + AES256CBCEncrypt enc(vchKey.data(), vchIV.data(), true); size_t nLen = enc.Encrypt(&vchPlaintext[0], vchPlaintext.size(), &vchCiphertext[0]); if(nLen < vchPlaintext.size()) return false; @@ -101,7 +101,7 @@ bool CCrypter::Decrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingM vchPlaintext.resize(nLen); - AES256CBCDecrypt dec(chKey, chIV, true); + AES256CBCDecrypt dec(vchKey.data(), vchIV.data(), true); nLen = dec.Decrypt(&vchCiphertext[0], vchCiphertext.size(), &vchPlaintext[0]); if(nLen == 0) return false; diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 5d0a4a3305..f00f7fa731 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -77,8 +77,8 @@ class CCrypter { friend class wallet_crypto::TestCrypter; // for test access to chKey/chIV private: - unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; - unsigned char chIV[WALLET_CRYPTO_IV_SIZE]; + std::vector<unsigned char, secure_allocator<unsigned char>> vchKey; + std::vector<unsigned char, secure_allocator<unsigned char>> vchIV; bool fKeySet; int BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const; @@ -91,28 +91,21 @@ public: void CleanKey() { - memory_cleanse(chKey, sizeof(chKey)); - memory_cleanse(chIV, sizeof(chIV)); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); fKeySet = false; } CCrypter() { fKeySet = false; - - // Try to keep the key data out of swap (and be a bit over-careful to keep the IV that we don't even use out of swap) - // Note that this does nothing about suspend-to-disk (which will put all our key data on disk) - // Note as well that at no point in this program is any attempt made to prevent stealing of keys by reading the memory of the running process. - LockedPageManager::Instance().LockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().LockRange(&chIV[0], sizeof chIV); + vchKey.resize(WALLET_CRYPTO_KEY_SIZE); + vchIV.resize(WALLET_CRYPTO_IV_SIZE); } ~CCrypter() { CleanKey(); - - LockedPageManager::Instance().UnlockRange(&chKey[0], sizeof chKey); - LockedPageManager::Instance().UnlockRange(&chIV[0], sizeof chIV); } }; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index a3dd8b69b6..c5feb47899 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -808,7 +808,7 @@ UniValue processImport(const UniValue& data) { CBitcoinAddress pubKeyAddress = CBitcoinAddress(pubKey.GetID()); // Consistency check. - if (!isScript && pubKeyAddress.Get() != address.Get()) { + if (!isScript && !(pubKeyAddress.Get() == address.Get())) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); } @@ -819,7 +819,7 @@ UniValue processImport(const UniValue& data) { if (ExtractDestination(script, destination)) { scriptAddress = CBitcoinAddress(destination); - if (scriptAddress.Get() != pubKeyAddress.Get()) { + if (!(scriptAddress.Get() == pubKeyAddress.Get())) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); } } @@ -881,7 +881,7 @@ UniValue processImport(const UniValue& data) { CBitcoinAddress pubKeyAddress = CBitcoinAddress(pubKey.GetID()); // Consistency check. - if (!isScript && pubKeyAddress.Get() != address.Get()) { + if (!isScript && !(pubKeyAddress.Get() == address.Get())) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); } @@ -892,7 +892,7 @@ UniValue processImport(const UniValue& data) { if (ExtractDestination(script, destination)) { scriptAddress = CBitcoinAddress(destination); - if (scriptAddress.Get() != pubKeyAddress.Get()) { + if (!(scriptAddress.Get() == pubKeyAddress.Get())) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Consistency check failed"); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ab7e74988a..5a22e0278d 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -6,6 +6,7 @@ #include "amount.h" #include "base58.h" #include "chain.h" +#include "consensus/validation.h" #include "core_io.h" #include "init.h" #include "main.h" @@ -362,11 +363,14 @@ static void SendMoney(const CTxDestination &address, CAmount nValue, bool fSubtr vecSend.push_back(recipient); if (!pwalletMain->CreateTransaction(vecSend, wtxNew, reservekey, nFeeRequired, nChangePosRet, strError)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > pwalletMain->GetBalance()) - strError = strprintf("Error: This transaction requires a transaction fee of at least %s because of its amount, complexity, or use of recently received funds!", FormatMoney(nFeeRequired)); + strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); + throw JSONRPCError(RPC_WALLET_ERROR, strError); + } + CValidationState state; + if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get(), state)) { + strError = strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason()); throw JSONRPCError(RPC_WALLET_ERROR, strError); } - if (!pwalletMain->CommitTransaction(wtxNew, reservekey, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Error: The transaction was rejected! This might happen if some of the coins in your wallet were already spent, such as if you used a copy of the wallet and coins were spent in the copy but not marked as spent here."); } UniValue sendtoaddress(const JSONRPCRequest& request) @@ -959,8 +963,11 @@ UniValue sendmany(const JSONRPCRequest& request) bool fCreated = pwalletMain->CreateTransaction(vecSend, wtx, keyChange, nFeeRequired, nChangePosRet, strFailReason); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); - if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get())) - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction commit failed"); + CValidationState state; + if (!pwalletMain->CommitTransaction(wtx, keyChange, g_connman.get(), state)) { + strFailReason = strprintf("Transaction commit failed:: %s", state.GetRejectReason()); + throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + } return wtx.GetHash().GetHex(); } @@ -2279,7 +2286,7 @@ UniValue getwalletinfo(const JSONRPCRequest& request) " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index c5f55ef5f0..ce35c53c48 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -97,10 +97,10 @@ static void TestPassphraseSingle(const std::vector<unsigned char>& vchSalt, cons OldSetKeyFromPassphrase(passphrase, vchSalt, rounds, 0, chKey, chIV); - BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.chKey, sizeof(chKey)) == 0, \ - HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.chKey, crypt.chKey + (sizeof crypt.chKey))); - BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.chIV, sizeof(chIV)) == 0, \ - HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.chIV, crypt.chIV + (sizeof crypt.chIV))); + BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.vchKey.data(), crypt.vchKey.size()) == 0, \ + HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.vchKey)); + BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.vchIV.data(), crypt.vchIV.size()) == 0, \ + HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.vchIV)); if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(chKey, &correctKey[0], sizeof(chKey)) == 0, \ @@ -127,7 +127,7 @@ static void TestDecrypt(const CCrypter& crypt, const std::vector<unsigned char>& CKeyingMaterial vchDecrypted2; int result1, result2; result1 = crypt.Decrypt(vchCiphertext, vchDecrypted1); - result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.chKey, crypt.chIV); + result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); // These two should be equal. However, OpenSSL 1.0.1j introduced a change @@ -152,7 +152,7 @@ static void TestEncryptSingle(const CCrypter& crypt, const CKeyingMaterial& vchP std::vector<unsigned char> vchCiphertext2; int result1 = crypt.Encrypt(vchPlaintext, vchCiphertext1); - int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.chKey, crypt.chIV); + int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.vchKey.data(), crypt.vchIV.data()); BOOST_CHECK(result1 == result2); BOOST_CHECK(vchCiphertext1 == vchCiphertext2); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c7f98b238e..c2bac6e330 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -8,7 +8,7 @@ #include "base58.h" #include "checkpoints.h" #include "chain.h" -#include "coincontrol.h" +#include "wallet/coincontrol.h" #include "consensus/consensus.h" #include "consensus/validation.h" #include "key.h" @@ -658,8 +658,79 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { + LOCK(cs_wallet); CWalletDB walletdb(strWalletFile); - return walletdb.ReorderTransactions(this); + + // Old wallets didn't have any defined order for transactions + // Probably a bad idea to change the output of this + + // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. + typedef pair<CWalletTx*, CAccountingEntry*> TxPair; + typedef multimap<int64_t, TxPair > TxItems; + TxItems txByTime; + + for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + { + CWalletTx* wtx = &((*it).second); + txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); + } + list<CAccountingEntry> acentries; + walletdb.ListAccountCreditDebit("", acentries); + BOOST_FOREACH(CAccountingEntry& entry, acentries) + { + txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); + } + + nOrderPosNext = 0; + std::vector<int64_t> nOrderPosOffsets; + for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) + { + CWalletTx *const pwtx = (*it).second.first; + CAccountingEntry *const pacentry = (*it).second.second; + int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; + + if (nOrderPos == -1) + { + nOrderPos = nOrderPosNext++; + nOrderPosOffsets.push_back(nOrderPos); + + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + else + { + int64_t nOrderPosOff = 0; + BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) + { + if (nOrderPos >= nOffsetStart) + ++nOrderPosOff; + } + nOrderPos += nOrderPosOff; + nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); + + if (!nOrderPosOff) + continue; + + // Since we're changing the order, write it back + if (pwtx) + { + if (!walletdb.WriteTx(*pwtx)) + return DB_LOAD_FAIL; + } + else + if (!walletdb.WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) + return DB_LOAD_FAIL; + } + } + walletdb.WriteOrderPosNext(nOrderPosNext); + + return DB_LOAD_OK; } int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) @@ -1463,7 +1534,8 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = *(item.second); LOCK(mempool.cs); - wtx.AcceptToMemoryPool(maxTxFee); + CValidationState state; + wtx.AcceptToMemoryPool(maxTxFee, state); } } @@ -2433,17 +2505,22 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt dPriority = wtxNew.ComputePriority(dPriority, nBytes); + // Allow to override the default confirmation target over the CoinControl instance + int currentConfirmationTarget = nTxConfirmTarget; + if (coinControl && coinControl->nConfirmTarget > 0) + currentConfirmationTarget = coinControl->nConfirmTarget; + // Can we complete this as a free transaction? if (fSendFreeTransactions && nBytes <= MAX_FREE_TRANSACTION_CREATE_SIZE) { // Not enough fee: enough priority? - double dPriorityNeeded = mempool.estimateSmartPriority(nTxConfirmTarget); + double dPriorityNeeded = mempool.estimateSmartPriority(currentConfirmationTarget); // Require at least hard-coded AllowFree. if (dPriority >= dPriorityNeeded && AllowFree(dPriority)) break; } - CAmount nFeeNeeded = GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2474,7 +2551,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman) +bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state) { { LOCK2(cs_main, cs_wallet); @@ -2502,9 +2579,9 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee)) { + if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { // This must not fail. The transaction has already been signed and recorded. - LogPrintf("CommitTransaction(): Error: Transaction not valid\n"); + LogPrintf("CommitTransaction(): Error: Transaction not valid, %s\n", state.GetRejectReason()); return false; } wtxNew.RelayWalletTransaction(connman); @@ -3472,6 +3549,16 @@ bool CWallet::InitLoadWallet() return true; } +void CWallet::postInitProcess(boost::thread_group& threadGroup) +{ + // Add wallet transactions that aren't already in a block to mempool + // Do this here as mempool requires genesis block to be loaded + ReacceptWalletTransactions(); + + // Run a thread to flush wallet periodically + threadGroup.create_thread(boost::bind(&ThreadFlushWalletDB, boost::ref(this->strWalletFile))); +} + bool CWallet::ParameterInteraction() { if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) @@ -3649,8 +3736,7 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee) +bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - CValidationState state; return ::AcceptToMemoryPool(mempool, state, *this, true, NULL, false, nAbsurdFee); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 3b37f7cb1f..57b17d87ad 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -27,6 +27,7 @@ #include <vector> #include <boost/shared_ptr.hpp> +#include <boost/thread.hpp> extern CWallet* pwalletMain; @@ -53,7 +54,7 @@ static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -sendfreetransactions static const bool DEFAULT_SEND_FREE_TRANSACTIONS = false; //! -txconfirmtarget default -static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 2; +static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default static const bool DEFAULT_WALLET_RBF = false; //! Largest (in bytes) free transaction we're willing to create @@ -133,7 +134,7 @@ struct CRecipient typedef std::map<std::string, std::string> mapValue_t; -static void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) +static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) { if (!mapValue.count("n")) { @@ -144,7 +145,7 @@ static void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) } -static void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) +static inline void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) { if (nOrderPos == -1) return; @@ -215,7 +216,7 @@ public: bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee); + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -774,7 +775,7 @@ public: */ bool CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletTx& wtxNew, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl *coinControl = NULL, bool sign = true); - bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman); + bool CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CConnman* connman, CValidationState& state); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries); bool AddAccountingEntry(const CAccountingEntry&); @@ -912,6 +913,12 @@ public: /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ static bool InitLoadWallet(); + /** + * Wallet post-init setup + * Gives the wallet a chance to register repetitive tasks and complete post-init tasks + */ + void postInitProcess(boost::thread_group& threadGroup); + /* Wallets parameter interaction */ static bool ParameterInteraction(); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 80bfe8255d..43fd6a20ad 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -251,82 +251,6 @@ void CWalletDB::ListAccountCreditDebit(const string& strAccount, list<CAccountin pcursor->close(); } -DBErrors CWalletDB::ReorderTransactions(CWallet* pwallet) -{ - LOCK(pwallet->cs_wallet); - // Old wallets didn't have any defined order for transactions - // Probably a bad idea to change the output of this - - // First: get all CWalletTx and CAccountingEntry into a sorted-by-time multimap. - typedef pair<CWalletTx*, CAccountingEntry*> TxPair; - typedef multimap<int64_t, TxPair > TxItems; - TxItems txByTime; - - for (map<uint256, CWalletTx>::iterator it = pwallet->mapWallet.begin(); it != pwallet->mapWallet.end(); ++it) - { - CWalletTx* wtx = &((*it).second); - txByTime.insert(make_pair(wtx->nTimeReceived, TxPair(wtx, (CAccountingEntry*)0))); - } - list<CAccountingEntry> acentries; - ListAccountCreditDebit("", acentries); - BOOST_FOREACH(CAccountingEntry& entry, acentries) - { - txByTime.insert(make_pair(entry.nTime, TxPair((CWalletTx*)0, &entry))); - } - - int64_t& nOrderPosNext = pwallet->nOrderPosNext; - nOrderPosNext = 0; - std::vector<int64_t> nOrderPosOffsets; - for (TxItems::iterator it = txByTime.begin(); it != txByTime.end(); ++it) - { - CWalletTx *const pwtx = (*it).second.first; - CAccountingEntry *const pacentry = (*it).second.second; - int64_t& nOrderPos = (pwtx != 0) ? pwtx->nOrderPos : pacentry->nOrderPos; - - if (nOrderPos == -1) - { - nOrderPos = nOrderPosNext++; - nOrderPosOffsets.push_back(nOrderPos); - - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - else - { - int64_t nOrderPosOff = 0; - BOOST_FOREACH(const int64_t& nOffsetStart, nOrderPosOffsets) - { - if (nOrderPos >= nOffsetStart) - ++nOrderPosOff; - } - nOrderPos += nOrderPosOff; - nOrderPosNext = std::max(nOrderPosNext, nOrderPos + 1); - - if (!nOrderPosOff) - continue; - - // Since we're changing the order, write it back - if (pwtx) - { - if (!WriteTx(*pwtx)) - return DB_LOAD_FAIL; - } - else - if (!WriteAccountingEntry(pacentry->nEntryNo, *pacentry)) - return DB_LOAD_FAIL; - } - } - WriteOrderPosNext(nOrderPosNext); - - return DB_LOAD_OK; -} - class CWalletScanState { public: unsigned int nKeys; @@ -711,7 +635,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) WriteVersion(CLIENT_VERSION); if (wss.fAnyUnordered) - result = ReorderTransactions(pwallet); + result = pwallet->ReorderTransactions(); pwallet->laccentries.clear(); ListAccountCreditDebit("*", pwallet->laccentries); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 9c9d4922a4..a0525bd9a7 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -153,6 +153,7 @@ public: /// This writes directly to the database, and will not update the CWallet's cached accounting entries! /// Use wallet.AddAccountingEntry instead, to write *and* update its caches. + bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); bool WriteAccountingEntry_Backend(const CAccountingEntry& acentry); bool ReadAccount(const std::string& strAccount, CAccount& account); bool WriteAccount(const std::string& strAccount, const CAccount& account); @@ -165,7 +166,6 @@ public: CAmount GetAccountCreditDebit(const std::string& strAccount); void ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& acentries); - DBErrors ReorderTransactions(CWallet* pwallet); DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(CWallet* pwallet, std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(CWallet* pwallet, std::vector<CWalletTx>& vWtx); @@ -180,7 +180,6 @@ private: CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); - bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); }; void ThreadFlushWalletDB(const std::string& strFile); |