diff options
70 files changed, 1807 insertions, 209 deletions
diff --git a/.gitignore b/.gitignore index a8035731d1..a8722aa593 100644 --- a/.gitignore +++ b/.gitignore @@ -85,15 +85,13 @@ src/test/buildenv.py # Resources cpp qrc_*.cpp -# Qt creator -*.pro.user - # Mac specific .DS_Store build #lcov *.gcno +*.gcda /*.info test_bitcoin.coverage/ total.coverage/ @@ -107,6 +105,9 @@ qa/pull-tester/run-bitcoind-for-test.sh qa/pull-tester/tests_config.py qa/pull-tester/cache/* qa/pull-tester/test.*/* +qa/tmp +cache/ +share/BitcoindComparisonTool.jar !src/leveldb*/Makefile diff --git a/Makefile.am b/Makefile.am index 8a7140398f..f0961c64ec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,7 +39,7 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) $ COVERAGE_INFO = baseline_filtered_combined.info baseline.info block_test.info \ leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \ - baseline_filtered.info block_test_filtered.info \ + baseline_filtered.info block_test_filtered.info rpc_test.info rpc_test_filtered.info \ leveldb_baseline_filtered.info test_bitcoin_coverage.info test_bitcoin.info dist-hook: @@ -170,7 +170,7 @@ test_bitcoin_filtered.info: test_bitcoin.info block_test.info: test_bitcoin_filtered.info $(MKDIR_P) qa/tmp - -@TIMEOUT=15 qa/pull-tester/run-bitcoind-for-test.sh $(JAVA) -jar $(JAVA_COMPARISON_TOOL) qa/tmp/compTool 0 + -@TIMEOUT=15 qa/pull-tester/run-bitcoind-for-test.sh $(JAVA) -jar $(JAVA_COMPARISON_TOOL) qa/tmp/compTool $(COMPARISON_TOOL_REORG_TESTS) $(LCOV) -c -d $(abs_builddir)/src --t BitcoinJBlockTest -o $@ $(LCOV) -z -d $(abs_builddir)/src $(LCOV) -z -d $(abs_builddir)/src/leveldb @@ -178,11 +178,20 @@ block_test.info: test_bitcoin_filtered.info block_test_filtered.info: block_test.info $(LCOV) -r $< "/usr/include/*" -o $@ +rpc_test.info: test_bitcoin_filtered.info + -@TIMEOUT=15 python qa/pull-tester/rpc-tests.py $(EXTENDED_RPC_TESTS) + $(LCOV) -c -d $(abs_builddir)/src --t rpc-tests -o $@ + $(LCOV) -z -d $(abs_builddir)/src + $(LCOV) -z -d $(abs_builddir)/src/leveldb + +rpc_test_filtered.info: rpc_test.info + $(LCOV) -r $< "/usr/include/*" -o $@ + test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ -total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info block_test_filtered.info - $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a block_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt +total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info block_test_filtered.info rpc_test_filtered.info + $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a block_test_filtered.info -a rpc_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin.coverage/.dirstamp: test_bitcoin_coverage.info $(GENHTML) -s $< -o $(@D) @@ -211,4 +220,4 @@ CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) .INTERMEDIATE: $(COVERAGE_INFO) clean-local: - rm -rf test_bitcoin.coverage/ total.coverage/ $(OSX_APP) + rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP) diff --git a/configure.ac b/configure.ac index 4318aafa55..25ace88f32 100644 --- a/configure.ac +++ b/configure.ac @@ -58,6 +58,7 @@ AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) AC_PATH_PROG(JAVA, java) +AC_PATH_PROG(PYTHON, python) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) @@ -106,6 +107,11 @@ AC_ARG_ENABLE([comparison-tool-reorg-tests], [use_comparison_tool_reorg_tests=$enableval], [use_comparison_tool_reorg_tests=no]) +AC_ARG_ENABLE([extended-rpc-tests], + AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]), + [use_extended_rpc_tests=$enableval], + [use_extended_rpc_tests=no]) + AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], @@ -341,6 +347,10 @@ else AC_SUBST(COMPARISON_TOOL_REORG_TESTS, 0) fi +if test x$use_extended_rpc_tests != xno; then + AC_SUBST(EXTENDED_RPC_TESTS, -extended) +fi + if test x$use_lcov = xyes; then if test x$LCOV = x; then AC_MSG_ERROR("lcov testing requested but lcov not found") @@ -351,6 +361,9 @@ if test x$use_lcov = xyes; then if test x$JAVA = x; then AC_MSG_ERROR("lcov testing requested but java not found") fi + if test x$PYTHON = x; then + AC_MSG_ERROR("lcov testing requested but python not found") + fi if test x$GENHTML = x; then AC_MSG_ERROR("lcov testing requested but genhtml not found") fi diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index f90afa7f20..278794f14c 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -1,9 +1,14 @@ Contents -=========== +======== This directory contains tools for developers working on this repository. +clang-format.py +=============== + +A script to format cpp source code according to [.clang-format](../../src/.clang-format). This should only be applied to new files or files which are currently not actively developed on. Also, git subtrees are not subject to formatting. + github-merge.sh -================== +=============== A small script to automate merging pull-requests securely and sign them with GPG. @@ -37,23 +42,31 @@ Configuring the github-merge tool for the bitcoin repository is done in the foll git config --global user.signingkey mykeyid (if you want to GPG sign) fix-copyright-headers.py -=========================== +======================== Every year newly updated files need to have its copyright headers updated to reflect the current year. If you run this script from src/ it will automatically update the year on the copyright header for all .cpp and .h files if these have a git commit from the current year. For example a file changed in 2014 (with 2014 being the current year): + ```// Copyright (c) 2009-2013 The Bitcoin Core developers``` would be changed to: + ```// Copyright (c) 2009-2014 The Bitcoin Core developers``` +optimize-pngs.py +================ + +A script to optimize png files in the bitcoin +repository (requires pngcrush). + symbol-check.py -================== +=============== A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are +allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported Linux distribution versions. Example usage after a gitian build: @@ -70,7 +83,7 @@ If there are 'unsupported' symbols, the return value will be 1 a list like this .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15 update-translations.py -======================= +====================== Run this script from the root of the repository to update all translations from transifex. It will do the following automatically: @@ -93,4 +106,5 @@ maintained: * for sec/leveldb: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork) Usage: git-subtree-check.sh DIR COMMIT + COMMIT may be omitted, in which case HEAD is used. diff --git a/contrib/devtools/clang-format.py b/contrib/devtools/clang-format.py new file mode 100755 index 0000000000..cee99047ac --- /dev/null +++ b/contrib/devtools/clang-format.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +''' +Wrapper script for clang-format + +Copyright (c) 2015 MarcoFalke +Copyright (c) 2015 The Bitcoin Core developers +Distributed under the MIT software license, see the accompanying +file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +import os +import sys +import subprocess + +tested_versions = ['3.6.0', '3.6.1', '3.6.2'] # A set of versions known to produce the same output +accepted_file_extensions = ('.h', '.cpp') # Files to format + +def check_clang_format_version(clang_format_exe): + try: + output = subprocess.check_output([clang_format_exe, '-version']) + for ver in tested_versions: + if ver in output: + print "Detected clang-format version " + ver + return + raise RuntimeError("Untested version: " + output) + except Exception as e: + print 'Could not verify version of ' + clang_format_exe + '.' + raise e + +def check_command_line_args(argv): + required_args = ['{clang-format-exe}', '{files}'] + example_args = ['clang-format-3.x', 'src/main.cpp', 'src/wallet/*'] + + if(len(argv) < len(required_args) + 1): + for word in (['Usage:', argv[0]] + required_args): + print word, + print '' + for word in (['E.g:', argv[0]] + example_args): + print word, + print '' + sys.exit(1) + +def run_clang_format(clang_format_exe, files): + for target in files: + if os.path.isdir(target): + for path, dirs, files in os.walk(target): + run_clang_format(clang_format_exe, (os.path.join(path, f) for f in files)) + elif target.endswith(accepted_file_extensions): + print "Format " + target + subprocess.check_call([clang_format_exe, '-i', '-style=file', target], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT) + else: + print "Skip " + target + +def main(argv): + check_command_line_args(argv) + clang_format_exe = argv[1] + files = argv[2:] + check_clang_format_version(clang_format_exe) + run_clang_format(clang_format_exe, files) + +if __name__ == "__main__": + main(sys.argv) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py new file mode 100755 index 0000000000..e96eaa9c38 --- /dev/null +++ b/contrib/devtools/security-check.py @@ -0,0 +1,181 @@ +#!/usr/bin/python2 +''' +Perform basic ELF security checks on a series of executables. +Exit status will be 0 if succesful, and the program will be silent. +Otherwise the exit status will be 1 and it will log which executables failed which checks. +Needs `readelf` (for ELF) and `objdump` (for PE). +''' +from __future__ import division,print_function +import subprocess +import sys +import os + +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') + +def check_ELF_PIE(executable): + ''' + Check for position independent executable (PIE), allowing for address space randomization. + ''' + p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + ok = False + for line in stdout.split('\n'): + line = line.split() + if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': + ok = True + return ok + +def get_ELF_program_headers(executable): + '''Return type and flags for ELF program headers''' + p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + in_headers = False + count = 0 + headers = [] + for line in stdout.split('\n'): + if line.startswith('Program Headers:'): + in_headers = True + if line == '': + in_headers = False + if in_headers: + if count == 1: # header line + ofs_typ = line.find('Type') + ofs_offset = line.find('Offset') + ofs_flags = line.find('Flg') + ofs_align = line.find('Align') + if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: + raise ValueError('Cannot parse elfread -lW output') + elif count > 1: + typ = line[ofs_typ:ofs_offset].rstrip() + flags = line[ofs_flags:ofs_align].rstrip() + headers.append((typ, flags)) + count += 1 + return headers + +def check_ELF_NX(executable): + ''' + Check that no sections are writable and executable (including the stack) + ''' + have_wx = False + have_gnu_stack = False + for (typ, flags) in get_ELF_program_headers(executable): + if typ == 'GNU_STACK': + have_gnu_stack = True + if 'W' in flags and 'E' in flags: # section is both writable and executable + have_wx = True + return have_gnu_stack and not have_wx + +def check_ELF_RELRO(executable): + ''' + Check for read-only relocations. + GNU_RELRO program header must exist + Dynamic section must have BIND_NOW flag + ''' + have_gnu_relro = False + for (typ, flags) in get_ELF_program_headers(executable): + # Note: not checking flags == 'R': here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + # However, the dynamic linker need to write to this area so these are RW. + # Glibc itself takes care of mprotecting this area R after relocations are finished. + # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 + if typ == 'GNU_RELRO': + have_gnu_relro = True + + have_bindnow = False + p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>1 and tokens[1] == '(BIND_NOW)': + have_bindnow = True + return have_gnu_relro and have_bindnow + +def check_ELF_Canary(executable): + ''' + Check for use of stack canary + ''' + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + ok = False + for line in stdout.split('\n'): + if '__stack_chk_fail' in line: + ok = True + return ok + +def get_PE_dll_characteristics(executable): + ''' + Get PE DllCharacteristics bits + ''' + p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': + return int(tokens[1],16) + return 0 + + +def check_PE_PIE(executable): + '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + return bool(get_PE_dll_characteristics(executable) & 0x40) + +def check_PE_NX(executable): + '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' + return bool(get_PE_dll_characteristics(executable) & 0x100) + +CHECKS = { +'ELF': [ + ('PIE', check_ELF_PIE), + ('NX', check_ELF_NX), + ('RELRO', check_ELF_RELRO), + ('Canary', check_ELF_Canary) +], +'PE': [ + ('PIE', check_PE_PIE), + ('NX', check_PE_NX) +] +} + +def identify_executable(executable): + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + return None + +if __name__ == '__main__': + retval = 0 + for filename in sys.argv[1:]: + try: + etype = identify_executable(filename) + if etype is None: + print('%s: unknown format' % filename) + retval = 1 + continue + + failed = [] + for (name, func) in CHECKS[etype]: + if not func(filename): + failed.append(name) + if failed: + print('%s: failed %s' % (filename, ' '.join(failed))) + retval = 1 + except IOError: + print('%s: cannot open' % filename) + retval = 1 + exit(retval) + diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index fad891f800..34f1ed2d17 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ from __future__ import division, print_function import subprocess import re import sys +import os # Debian 6.0.9 (Squeeze) has: # @@ -45,8 +46,10 @@ MAX_VERSIONS = { IGNORE_EXPORTS = { '_edata', '_end', '_init', '__bss_start', '_fini' } -READELF_CMD = '/usr/bin/readelf' -CPPFILT_CMD = '/usr/bin/c++filt' +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = {'librt.so.1','libpthread.so.0','libanl.so.1','libm.so.6','libgcc_s.so.1','libc.so.6','ld-linux-x86-64.so.2'} class CPPFilt(object): ''' @@ -98,6 +101,22 @@ def check_version(max_versions, version): return False return ver <= max_versions[lib] +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 @@ -113,6 +132,11 @@ if __name__ == '__main__': continue print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py new file mode 100755 index 0000000000..fed7626aab --- /dev/null +++ b/contrib/devtools/test-security-check.py @@ -0,0 +1,60 @@ +#!/usr/bin/python2 +''' +Test script for security-check.py +''' +from __future__ import division,print_function +import subprocess +import sys +import unittest + +def write_testcode(filename): + with open(filename, 'w') as f: + f.write(''' + #include <stdio.h> + int main() + { + printf("the quick brown fox jumps over the lazy god\\n"); + return 0; + } + ''') + +def call_security_check(cc, source, executable, options): + subprocess.check_call([cc,source,'-o',executable] + options) + p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + return (p.returncode, stdout.rstrip()) + +class TestSecurityChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = 'gcc' + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + (0, '')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = 'i686-w64-mingw32-gcc' + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, []), + (1, executable+': failed PIE NX')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), + (0, '')) + +if __name__ == '__main__': + unittest.main() + diff --git a/contrib/gitian-downloader/sipa-key.pgp b/contrib/gitian-downloader/sipa-key.pgp Binary files differindex ffa09bb4ad..a1930ddee9 100644 --- a/contrib/gitian-downloader/sipa-key.pgp +++ b/contrib/gitian-downloader/sipa-key.pgp diff --git a/contrib/init/README.md b/contrib/init/README.md index 0d19da3039..eb5d30acce 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -5,6 +5,7 @@ Upstart: bitcoind.conf OpenRC: bitcoind.openrc bitcoind.openrcconf CentOS: bitcoind.init +OS X: org.bitcoin.bitcoind.plist have been made available to assist packagers in creating node packages here. diff --git a/contrib/init/org.bitcoin.bitcoind.plist b/contrib/init/org.bitcoin.bitcoind.plist new file mode 100644 index 0000000000..e94cd4466d --- /dev/null +++ b/contrib/init/org.bitcoin.bitcoind.plist @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>Label</key> + <string>org.bitcoin.bitcoind</string> + <key>ProgramArguments</key> + <array> + <string>/usr/local/bin/bitcoind</string> + <string>-daemon</string> + </array> + <key>RunAtLoad</key> + <true/> +</dict> +</plist> diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits new file mode 100644 index 0000000000..31aeb8f3d3 --- /dev/null +++ b/contrib/verify-commits/allow-revsig-commits @@ -0,0 +1,2 @@ +586a29253dabec3ca0f1ccba9091daabd16b8411 +eddaba7b5692288087a926da5733e86b47274e4e diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 6b5137e7b5..0218b82e11 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -1,15 +1,33 @@ #!/bin/sh INPUT=$(</dev/stdin) VALID=false +REVSIG=false IFS=$'\n' for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do - case "$LINE" in "[GNUPG:] VALIDSIG"*) + case "$LINE" in + "[GNUPG:] VALIDSIG "*) while read KEY; do case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac done < ./contrib/verify-commits/trusted-keys + ;; + "[GNUPG:] REVKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + while read KEY; do + case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY:24:40} "*) + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${KEY:24:40} " + ;; + esac + done < ./contrib/verify-commits/trusted-keys + ;; esac done if ! $VALID; then exit 1 fi -echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null +if $VALID && $REVSIG; then + echo "$INPUT" | gpg --trust-model always "$@" | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" 2>/dev/null + echo "$GOODREVSIG" +else + echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null +fi diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index 607c0cac45..c57222818a 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,5 +1,5 @@ #!/bin/bash -if ! [[ "$2" =~ [git@]?[www.]?github.com[:|/]bitcoin/bitcoin[.git]? ]]; then +if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then exit 0 fi diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 658ad0375b..a0dce7a8a5 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -3,3 +3,4 @@ 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 AF8BE07C7049F3A26B239D5325B3083201782B2F 81291FA67D2C379A006A053FEAB5AF94D9E9ABE7 +133EAC179436F14A5CF1B794860FEB804E669320 diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index 5841fa2077..9ba781008a 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -7,11 +7,23 @@ git log "$DIR" VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") +IS_REVSIG_ALLOWED () { + while read LINE; do + [ "$LINE" = "$1" ] && return 0 + done < "${DIR}/allow-revsig-commits" + return 1 +} + HAVE_FAILED=false IS_SIGNED () { if [ $1 = $VERIFIED_ROOT ]; then return 0; fi + if IS_REVSIG_ALLOWED "$1"; then + export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 + else + export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 + fi if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then return 1; fi diff --git a/doc/bips.md b/doc/bips.md index c84bd966f5..c780e2dde0 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -14,6 +14,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.12.0**): * [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). * [`BIP 42`](https://github.com/bitcoin/bips/blob/master/bip-0042.mediawiki): The bug that would have caused the subsidy schedule to resume after block 13440000 was fixed in **v0.9.2** ([PR #3842](https://github.com/bitcoin/bitcoin/pull/3842)). * [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). +* [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, but only enforced for peer versions `>=70011` as of **v0.12.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579)). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 4189d22187..7fe292f1f8 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -171,3 +171,36 @@ Threads - BitcoinMiner : Generates bitcoins (if wallet is enabled). - Shutdown : Does an orderly shutdown of everything. + +Ignoring IDE/editor files +-------------------------- + +In closed-source environments in which everyone uses the same IDE it is common +to add temporary files it produces to the project-wide `.gitignore` file. + +However, in open source software such as Bitcoin Core, where everyone uses +their own editors/IDE/tools, it is less common. Only you know what files your +editor produces and this may change from version to version. The canonical way +to do this is thus to create your local gitignore. Add this to `~/.gitconfig`: + +``` +[core] + excludesfile = /home/.../.gitignore_global +``` + +(alternatively, type the command `git config --global core.excludesfile ~/.gitignore_global` +on a terminal) + +Then put your favourite tool's temporary filenames in that file, e.g. +``` +# NetBeans +nbproject/ +``` + +Another option is to create a per-repository excludes file `.git/info/exclude`. +These are not committed but apply only to one repository. + +If a set of tools is used by the build system or scripts the repository (for +example, lcov) it is perfectly acceptable to add its files to `.gitignore` +and commit them. + diff --git a/doc/init.md b/doc/init.md index d24c2d1dbf..e3db5b05ef 100644 --- a/doc/init.md +++ b/doc/init.md @@ -13,8 +13,9 @@ can be found in the contrib/init folder. 1. Service User --------------------------------- -All three startup configurations assume the existence of a "bitcoin" user +All three Linux startup configurations assume the existence of a "bitcoin" user and group. They must be created before attempting to use these scripts. +The OS X configuration assumes bitcoind will be set up for the current user. 2. Configuration --------------------------------- @@ -48,6 +49,8 @@ see `contrib/debian/examples/bitcoin.conf`. 3. Paths --------------------------------- +3a) Linux + All three configurations assume several paths that might need to be adjusted. Binary: `/usr/bin/bitcoind` @@ -62,6 +65,13 @@ reasons to make the configuration file and data directory only readable by the bitcoin user and group. Access to bitcoin-cli and other bitcoind rpc clients can then be controlled by group membership. +3b) Mac OS X + +Binary: `/usr/local/bin/bitcoind` +Configuration file: `~/Library/Application Support/Bitcoin/bitcoin.conf` +Data directory: `~/Library/Application Support/Bitcoin` +Lock file: `~/Library/Application Support/Bitcoin/.lock` + 4. Installing Service Configuration ----------------------------------- @@ -97,6 +107,17 @@ Using this script, you can adjust the path and flags to the bitcoind program by setting the BITCOIND and FLAGS environment variables in the file /etc/sysconfig/bitcoind. You can also use the DAEMONOPTS environment variable here. +4e) Mac OS X + +Copy org.bitcoin.bitcoind.plist into ~/Library/LaunchAgents. Load the launch agent by +running `launchctl load ~/Library/LaunchAgents/org.bitcoin.bitcoind.plist`. + +This Launch Agent will cause bitcoind to start whenever the user logs in. + +NOTE: This approach is intended for those wanting to run bitcoind as the current user. +You will need to modify org.bitcoin.bitcoind.plist if you intend to use it as a +Launch Daemon with a dedicated bitcoin user. + 5. Auto-respawn ----------------------------------- diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 8a431d718d..3059fee426 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -70,6 +70,8 @@ testScripts = [ 'blockchain.py', ] testScriptsExt = [ + 'bip65-cltv.py', + 'bip65-cltv-p2p.py', 'bipdersig-p2p.py', 'bipdersig.py', 'getblocktemplate_longpoll.py', @@ -82,12 +84,13 @@ testScriptsExt = [ 'keypool.py', 'receivedby.py', # 'rpcbind_test.py', #temporary, bug in libevent, see #6655 -# 'script_test.py', #used for manual comparison of 2 binaries +# 'script_test.py', #used for manual comparison of 2 binaries 'smartfees.py', 'maxblocksinflight.py', 'invalidblockrequest.py', 'p2p-acceptblock.py', 'mempool_packages.py', + 'maxuploadtarget.py', ] #Enable ZMQ tests diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py new file mode 100755 index 0000000000..1f8548c219 --- /dev/null +++ b/qa/rpc-tests/bip65-cltv-p2p.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python2 +# +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.util import * +from test_framework.mininode import CTransaction, NetworkThread +from test_framework.blocktools import create_coinbase, create_block +from test_framework.comptool import TestInstance, TestManager +from test_framework.script import CScript, OP_1NEGATE, OP_NOP2, OP_DROP +from binascii import hexlify, unhexlify +import cStringIO +import time + +def cltv_invalidate(tx): + '''Modify the signature in vin 0 of the tx to fail CLTV + + Prepends -1 CLTV DROP in the scriptSig itself. + ''' + tx.vin[0].scriptSig = CScript([OP_1NEGATE, OP_NOP2, OP_DROP] + + list(CScript(tx.vin[0].scriptSig))) + +''' +This test is meant to exercise BIP65 (CHECKLOCKTIMEVERIFY) +Connect to a single node. +Mine 2 (version 3) blocks (save the coinbases for later). +Generate 98 more version 3 blocks, verify the node accepts. +Mine 749 version 4 blocks, verify the node accepts. +Check that the new CLTV rules are not enforced on the 750th version 4 block. +Check that the new CLTV rules are enforced on the 751st version 4 block. +Mine 199 new version blocks. +Mine 1 old-version block. +Mine 1 new version block. +Mine 1 old version block, see that the node rejects. +''' + +class BIP65Test(ComparisonTestFramework): + + def __init__(self): + self.num_nodes = 1 + + def setup_network(self): + # Must set the blockversion for this test + self.nodes = start_nodes(1, self.options.tmpdir, + extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=3']], + binary=[self.options.testbinary]) + + def run_test(self): + test = TestManager(self, self.options.tmpdir) + test.add_all_connections(self.nodes) + NetworkThread().start() # Start up network handling in another thread + test.run() + + def create_transaction(self, node, coinbase, to_address, amount): + from_txid = node.getblock(coinbase)['tx'][0] + inputs = [{ "txid" : from_txid, "vout" : 0}] + outputs = { to_address : amount } + rawtx = node.createrawtransaction(inputs, outputs) + signresult = node.signrawtransaction(rawtx) + tx = CTransaction() + f = cStringIO.StringIO(unhexlify(signresult['hex'])) + tx.deserialize(f) + return tx + + def get_tests(self): + + self.coinbase_blocks = self.nodes[0].generate(2) + self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) + self.nodeaddress = self.nodes[0].getnewaddress() + self.last_block_time = time.time() + + ''' 98 more version 3 blocks ''' + test_blocks = [] + for i in xrange(98): + block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1) + block.nVersion = 3 + block.rehash() + block.solve() + test_blocks.append([block, True]) + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance(test_blocks, sync_every_block=False) + + ''' Mine 749 version 4 blocks ''' + test_blocks = [] + for i in xrange(749): + block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1) + block.nVersion = 4 + block.rehash() + block.solve() + test_blocks.append([block, True]) + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance(test_blocks, sync_every_block=False) + + ''' + Check that the new CLTV rules are not enforced in the 750th + version 3 block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[0], self.nodeaddress, 1.0) + cltv_invalidate(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(2), self.last_block_time + 1) + block.nVersion = 4 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance([[block, True]]) + + ''' + Check that the new CLTV rules are enforced in the 751st version 4 + block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + cltv_invalidate(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1) + block.nVersion = 4 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + + ''' Mine 199 new version blocks on last valid tip ''' + test_blocks = [] + for i in xrange(199): + block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1) + block.nVersion = 4 + block.rehash() + block.solve() + test_blocks.append([block, True]) + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance(test_blocks, sync_every_block=False) + + ''' Mine 1 old version block ''' + block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1) + block.nVersion = 3 + block.rehash() + block.solve() + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance([[block, True]]) + + ''' Mine 1 new version block ''' + block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1) + block.nVersion = 4 + block.rehash() + block.solve() + self.last_block_time += 1 + self.tip = block.sha256 + yield TestInstance([[block, True]]) + + ''' Mine 1 old version block, should be invalid ''' + block = create_block(self.tip, create_coinbase(1), self.last_block_time + 1) + block.nVersion = 3 + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + +if __name__ == '__main__': + BIP65Test().main() diff --git a/qa/rpc-tests/bip65-cltv.py b/qa/rpc-tests/bip65-cltv.py new file mode 100755 index 0000000000..e90e11e6a7 --- /dev/null +++ b/qa/rpc-tests/bip65-cltv.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python2 +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test the CHECKLOCKTIMEVERIFY (BIP65) soft-fork logic +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +import os +import shutil + +class BIP65Test(BitcoinTestFramework): + + def setup_network(self): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, [])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=3"])) + self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=4"])) + connect_nodes(self.nodes[1], 0) + connect_nodes(self.nodes[2], 0) + self.is_network_split = False + self.sync_all() + + def run_test(self): + cnt = self.nodes[0].getblockcount() + + # Mine some old-version blocks + self.nodes[1].generate(100) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 100): + raise AssertionError("Failed to mine 100 version=3 blocks") + + # Mine 750 new-version blocks + for i in xrange(15): + self.nodes[2].generate(50) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 850): + raise AssertionError("Failed to mine 750 version=4 blocks") + + # TODO: check that new CHECKLOCKTIMEVERIFY rules are not enforced + + # Mine 1 new-version block + self.nodes[2].generate(1) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 851): + raise AssertionFailure("Failed to mine a version=4 blocks") + + # TODO: check that new CHECKLOCKTIMEVERIFY rules are enforced + + # Mine 198 new-version blocks + for i in xrange(2): + self.nodes[2].generate(99) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 1049): + raise AssertionError("Failed to mine 198 version=4 blocks") + + # Mine 1 old-version block + self.nodes[1].generate(1) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 1050): + raise AssertionError("Failed to mine a version=3 block after 949 version=4 blocks") + + # Mine 1 new-version blocks + self.nodes[2].generate(1) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 1051): + raise AssertionError("Failed to mine a version=4 block") + + # Mine 1 old-version blocks + try: + self.nodes[1].generate(1) + raise AssertionError("Succeeded to mine a version=3 block after 950 version=4 blocks") + except JSONRPCException: + pass + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 1051): + raise AssertionError("Accepted a version=3 block after 950 version=4 blocks") + + # Mine 1 new-version blocks + self.nodes[2].generate(1) + self.sync_all() + if (self.nodes[0].getblockcount() != cnt + 1052): + raise AssertionError("Failed to mine a version=4 block") + +if __name__ == '__main__': + BIP65Test().main() diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index a5c98b777e..b7bfe36285 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -43,7 +43,7 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res[u'transactions'], 200) assert_equal(res[u'height'], 200) assert_equal(res[u'txouts'], 200) - assert_equal(res[u'bytes_serialized'], 13000), + assert_equal(res[u'bytes_serialized'], 13924), assert_equal(len(res[u'bestblock']), 64) assert_equal(len(res[u'hash_serialized']), 64) diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index b66533543d..7888114c54 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -97,5 +97,19 @@ class HTTPBasicsTest (BitcoinTestFramework): assert_equal('"error":null' in out1, True) assert_equal(conn.sock!=None, True) #connection must be closed because bitcoind should use keep-alive by default + # Check excessive request size + conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port) + conn.connect() + conn.request('GET', '/' + ('x'*1000), '', headers) + out1 = conn.getresponse() + assert_equal(out1.status, httplib.NOT_FOUND) + + conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port) + conn.connect() + conn.request('GET', '/' + ('x'*10000), '', headers) + out1 = conn.getresponse() + assert_equal(out1.status, httplib.BAD_REQUEST) + + if __name__ == '__main__': HTTPBasicsTest ().main () diff --git a/qa/rpc-tests/maxuploadtarget.py b/qa/rpc-tests/maxuploadtarget.py new file mode 100755 index 0000000000..67c4a50985 --- /dev/null +++ b/qa/rpc-tests/maxuploadtarget.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python2 +# +# Distributed under the MIT/X11 software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.comptool import wait_until +import time + +''' +Test behavior of -maxuploadtarget. + +* Verify that getdata requests for old blocks (>1week) are dropped +if uploadtarget has been reached. +* Verify that getdata requests for recent blocks are respecteved even +if uploadtarget has been reached. +* Verify that the upload counters are reset after 24 hours. +''' + +# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending +# p2p messages to a node, generating the messages in the main testing logic. +class TestNode(NodeConnCB): + def __init__(self): + NodeConnCB.__init__(self) + self.create_callback_map() + self.connection = None + self.ping_counter = 1 + self.last_pong = msg_pong() + self.block_receive_map = {} + + def add_connection(self, conn): + self.connection = conn + self.peer_disconnected = False + + def on_inv(self, conn, message): + pass + + # Track the last getdata message we receive (used in the test) + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_block(self, conn, message): + message.block.calc_sha256() + try: + self.block_receive_map[message.block.sha256] += 1 + except KeyError as e: + self.block_receive_map[message.block.sha256] = 1 + + # Spin until verack message is received from the node. + # We use this to signal that our test can begin. This + # is called from the testing thread, so it needs to acquire + # the global lock. + def wait_for_verack(self): + def veracked(): + return self.verack_received + return wait_until(veracked, timeout=10) + + def wait_for_disconnect(self): + def disconnected(): + return self.peer_disconnected + return wait_until(disconnected, timeout=10) + + # Wrapper for the NodeConn's send_message function + def send_message(self, message): + self.connection.send_message(message) + + def on_pong(self, conn, message): + self.last_pong = message + + def on_close(self, conn): + self.peer_disconnected = True + + # Sync up with the node after delivery of a block + def sync_with_ping(self, timeout=30): + def received_pong(): + return (self.last_pong.nonce == self.ping_counter) + self.connection.send_message(msg_ping(nonce=self.ping_counter)) + success = wait_until(received_pong, timeout) + self.ping_counter += 1 + return success + +class MaxUploadTest(BitcoinTestFramework): + def __init__(self): + self.utxo = [] + + # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create + # So we have big transactions and full blocks to fill up our block files + # create one script_pubkey + script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes + for i in xrange (512): + script_pubkey = script_pubkey + "01" + # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change + self.txouts = "81" + for k in xrange(128): + # add txout value + self.txouts = self.txouts + "0000000000000000" + # add length of script_pubkey + self.txouts = self.txouts + "fd0402" + # add script_pubkey + self.txouts = self.txouts + script_pubkey + + def add_options(self, parser): + parser.add_option("--testbinary", dest="testbinary", + default=os.getenv("BITCOIND", "bitcoind"), + help="bitcoind binary to test") + + def setup_chain(self): + initialize_chain_clean(self.options.tmpdir, 2) + + def setup_network(self): + # Start a node with maxuploadtarget of 200 MB (/24h) + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-maxuploadtarget=200", "-blockmaxsize=999000"])) + + def mine_full_block(self, node, address): + # Want to create a full block + # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit + for j in xrange(14): + if len(self.utxo) < 14: + self.utxo = node.listunspent() + inputs=[] + outputs = {} + t = self.utxo.pop() + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) + remchange = t["amount"] - Decimal("0.001000") + outputs[address]=remchange + # Create a basic transaction that will send change back to ourself after account for a fee + # And then insert the 128 generated transaction outs in the middle rawtx[92] is where the # + # of txouts is stored and is the only thing we overwrite from the original transaction + rawtx = node.createrawtransaction(inputs, outputs) + newtx = rawtx[0:92] + newtx = newtx + self.txouts + newtx = newtx + rawtx[94:] + # Appears to be ever so slightly faster to sign with SIGHASH_NONE + signresult = node.signrawtransaction(newtx,None,None,"NONE") + txid = node.sendrawtransaction(signresult["hex"], True) + # Mine a full sized block which will be these transactions we just created + node.generate(1) + + def run_test(self): + # Before we connect anything, we first set the time on the node + # to be in the past, otherwise things break because the CNode + # time counters can't be reset backward after initialization + old_time = int(time.time() - 2*60*60*24*7) + self.nodes[0].setmocktime(old_time) + + # Generate some old blocks + self.nodes[0].generate(130) + + # test_nodes[0] will only request old blocks + # test_nodes[1] will only request new blocks + # test_nodes[2] will test resetting the counters + test_nodes = [] + connections = [] + + for i in xrange(3): + test_nodes.append(TestNode()) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) + test_nodes[i].add_connection(connections[i]) + + NetworkThread().start() # Start up network handling in another thread + [x.wait_for_verack() for x in test_nodes] + + # Test logic begins here + + # Now mine a big block + self.mine_full_block(self.nodes[0], self.nodes[0].getnewaddress()) + + # Store the hash; we'll request this later + big_old_block = self.nodes[0].getbestblockhash() + old_block_size = self.nodes[0].getblock(big_old_block, True)['size'] + big_old_block = int(big_old_block, 16) + + # Advance to two days ago + self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) + + # Mine one more block, so that the prior block looks old + self.mine_full_block(self.nodes[0], self.nodes[0].getnewaddress()) + + # We'll be requesting this new block too + big_new_block = self.nodes[0].getbestblockhash() + new_block_size = self.nodes[0].getblock(big_new_block)['size'] + big_new_block = int(big_new_block, 16) + + # test_nodes[0] will test what happens if we just keep requesting the + # the same big old block too many times (expect: disconnect) + + getdata_request = msg_getdata() + getdata_request.inv.append(CInv(2, big_old_block)) + + max_bytes_per_day = 200*1024*1024 + max_bytes_available = max_bytes_per_day - 144*1000000 + success_count = max_bytes_available / old_block_size + + # 144MB will be reserved for relaying new blocks, so expect this to + # succeed for ~70 tries. + for i in xrange(success_count): + test_nodes[0].send_message(getdata_request) + test_nodes[0].sync_with_ping() + assert_equal(test_nodes[0].block_receive_map[big_old_block], i+1) + + assert_equal(len(self.nodes[0].getpeerinfo()), 3) + # At most a couple more tries should succeed (depending on how long + # the test has been running so far). + for i in xrange(3): + test_nodes[0].send_message(getdata_request) + test_nodes[0].wait_for_disconnect() + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + print "Peer 0 disconnected after downloading old block too many times" + + # Requesting the current block on test_nodes[1] should succeed indefinitely, + # even when over the max upload target. + # We'll try 200 times + getdata_request.inv = [CInv(2, big_new_block)] + for i in xrange(200): + test_nodes[1].send_message(getdata_request) + test_nodes[1].sync_with_ping() + assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + + print "Peer 1 able to repeatedly download new block" + + # But if test_nodes[1] tries for an old block, it gets disconnected too. + getdata_request.inv = [CInv(2, big_old_block)] + test_nodes[1].send_message(getdata_request) + test_nodes[1].wait_for_disconnect() + assert_equal(len(self.nodes[0].getpeerinfo()), 1) + + print "Peer 1 disconnected after trying to download old block" + + print "Advancing system time on node to clear counters..." + + # If we advance the time by 24 hours, then the counters should reset, + # and test_nodes[2] should be able to retrieve the old block. + self.nodes[0].setmocktime(int(time.time())) + test_nodes[2].sync_with_ping() + test_nodes[2].send_message(getdata_request) + test_nodes[2].sync_with_ping() + assert_equal(test_nodes[2].block_receive_map[big_old_block], 1) + + print "Peer 2 able to download old block" + + [c.disconnect_node() for c in connections] + +if __name__ == '__main__': + MaxUploadTest().main() diff --git a/src/Makefile.am b/src/Makefile.am index 0a16a863eb..f35b9dc898 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,7 +110,7 @@ BITCOIN_CORE_H = \ init.h \ key.h \ keystore.h \ - leveldbwrapper.h \ + dbwrapper.h \ limitedmap.h \ main.h \ memusage.h \ @@ -188,7 +188,7 @@ libbitcoin_server_a_SOURCES = \ httprpc.cpp \ httpserver.cpp \ init.cpp \ - leveldbwrapper.cpp \ + dbwrapper.cpp \ main.cpp \ merkleblock.cpp \ miner.cpp \ @@ -412,7 +412,19 @@ libbitcoinconsensus_la_CPPFLAGS = $(CRYPTO_CFLAGS) -I$(builddir)/obj -DBUILD_BIT endif # -CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a *.gcda *.gcno +CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a +CLEANFILES += *.gcda *.gcno +CLEANFILES += compat/*.gcda compat/*.gcno +CLEANFILES += consensus/*.gcda consensus/*.gcno +CLEANFILES += crypto/*.gcda crypto/*.gcno +CLEANFILES += policy/*.gcda policy/*.gcno +CLEANFILES += primitives/*.gcda primitives/*.gcno +CLEANFILES += script/*.gcda script/*.gcno +CLEANFILES += support/*.gcda support/*.gcno +CLEANFILES += univalue/*.gcda univalue/*.gcno +CLEANFILES += wallet/*.gcda wallet/*.gcno +CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno +CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = obj/build.h @@ -422,7 +434,7 @@ clean-local: -$(MAKE) -C leveldb clean -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean - rm -f leveldb/*/*.gcno leveldb/helpers/memenv/*.gcno + -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h .rc.o: diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0fe8ea42ff..f23a8f41fc 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -54,7 +54,7 @@ BITCOIN_TESTS =\ test/hash_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ - test/leveldbwrapper_tests.cpp \ + test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ diff --git a/src/amount.h b/src/amount.h index 90e6b5aa8e..a4c7764cda 100644 --- a/src/amount.h +++ b/src/amount.h @@ -51,6 +51,7 @@ public: friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; ADD_SERIALIZE_METHODS; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 364baf9d13..6f22c70494 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -67,7 +67,7 @@ static bool AppInitRPC(int argc, char* argv[]) // Parameters // ParseParameters(argc, argv); - if (argc<2 || mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) { + if (argc<2 || mapArgs.count("-?") || mapArgs.count("-h") || mapArgs.count("-help") || mapArgs.count("-version")) { std::string strUsage = _("Bitcoin Core RPC client version") + " " + FormatFullVersion() + "\n"; if (!mapArgs.count("-version")) { strUsage += "\n" + _("Usage:") + "\n" + diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 5630d20cc5..3330fe5d12 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -44,7 +44,7 @@ static bool AppInitRawTx(int argc, char* argv[]) fCreateBlank = GetBoolArg("-create", false); - if (argc<2 || mapArgs.count("-?") || mapArgs.count("-help")) + if (argc<2 || mapArgs.count("-?") || mapArgs.count("-h") || mapArgs.count("-help")) { // First part of help message is specific to this utility std::string strUsage = _("Bitcoin Core bitcoin-tx utility version") + " " + FormatFullVersion() + "\n\n" + diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 284a42b326..d2af897242 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -72,7 +72,7 @@ bool AppInit(int argc, char* argv[]) ParseParameters(argc, argv); // Process help and version before taking care about datadir - if (mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) + if (mapArgs.count("-?") || mapArgs.count("-h") || mapArgs.count("-help") || mapArgs.count("-version")) { std::string strUsage = _("Bitcoin Core Daemon") + " " + _("version") + " " + FormatFullVersion() + "\n"; diff --git a/src/compat.h b/src/compat.h index 5378c2c761..20c2a25143 100644 --- a/src/compat.h +++ b/src/compat.h @@ -38,6 +38,7 @@ #include <sys/types.h> #include <net/if.h> #include <netinet/in.h> +#include <netinet/tcp.h> #include <arpa/inet.h> #include <ifaddrs.h> #include <limits.h> diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index f937844e9f..6d6ce7e099 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -13,4 +13,10 @@ static const unsigned int MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50; /** Coinbase transaction outputs can only be spent after this number of new blocks (network rule) */ static const int COINBASE_MATURITY = 100; +/** Flags for LockTime() */ +enum { + /* Use GetMedianTimePast() instead of nTime for end point timestamp. */ + LOCKTIME_MEDIAN_TIME_PAST = (1 << 1), +}; + #endif // BITCOIN_CONSENSUS_CONSENSUS_H diff --git a/src/leveldbwrapper.cpp b/src/dbwrapper.cpp index 32c9345be5..b6307cf0bf 100644 --- a/src/leveldbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include "util.h" #include "random.h" @@ -15,18 +15,18 @@ #include <memenv.h> #include <stdint.h> -void HandleError(const leveldb::Status& status) throw(leveldb_error) +void HandleError(const leveldb::Status& status) throw(dbwrapper_error) { if (status.ok()) return; LogPrintf("%s\n", status.ToString()); if (status.IsCorruption()) - throw leveldb_error("Database corrupted"); + throw dbwrapper_error("Database corrupted"); if (status.IsIOError()) - throw leveldb_error("Database I/O error"); + throw dbwrapper_error("Database I/O error"); if (status.IsNotFound()) - throw leveldb_error("Database entry missing"); - throw leveldb_error("Unknown database error"); + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); } static leveldb::Options GetOptions(size_t nCacheSize) @@ -45,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) { penv = NULL; readoptions.verify_checksums = true; @@ -76,7 +76,7 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); if (!key_exists && obfuscate && IsEmpty()) { - // Initialize non-degenerate obfuscation if it won't upset + // Initialize non-degenerate obfuscation if it won't upset // existing, non-obfuscated data. std::vector<unsigned char> new_key = CreateObfuscateKey(); @@ -90,7 +90,7 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); } -CLevelDBWrapper::~CLevelDBWrapper() +CDBWrapper::~CDBWrapper() { delete pdb; pdb = NULL; @@ -102,7 +102,7 @@ CLevelDBWrapper::~CLevelDBWrapper() options.env = NULL; } -bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb_error) +bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) throw(dbwrapper_error) { leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); HandleError(status); @@ -113,15 +113,15 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb // // We must use a string constructor which specifies length so that we copy // past the null-terminator. -const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); +const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); -const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; +const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; /** - * Returns a string (consisting of 8 random bytes) suitable for use as an - * obfuscating XOR key. + * Returns a string (consisting of 8 random bytes) suitable for use as an + * obfuscating XOR key. */ -std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const +std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const { unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); @@ -129,26 +129,24 @@ std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const } -bool CLevelDBWrapper::IsEmpty() +bool CDBWrapper::IsEmpty() { - boost::scoped_ptr<CLevelDBIterator> it(NewIterator()); + boost::scoped_ptr<CDBIterator> it(NewIterator()); it->SeekToFirst(); return !(it->Valid()); } -const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const -{ - return obfuscate_key; +const std::vector<unsigned char>& CDBWrapper::GetObfuscateKey() const +{ + return obfuscate_key; } -std::string CLevelDBWrapper::GetObfuscateKeyHex() const -{ - return HexStr(obfuscate_key); +std::string CDBWrapper::GetObfuscateKeyHex() const +{ + return HexStr(obfuscate_key); } -CLevelDBIterator::~CLevelDBIterator() { delete piter; } -bool CLevelDBIterator::Valid() { return piter->Valid(); } -void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); } -void CLevelDBIterator::Next() { piter->Next(); } -void CLevelDBIterator::Prev() { piter->Prev(); } +CDBIterator::~CDBIterator() { delete piter; } +bool CDBIterator::Valid() { return piter->Valid(); } +void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CDBIterator::Next() { piter->Next(); } diff --git a/src/leveldbwrapper.h b/src/dbwrapper.h index 60101e18cc..aa28767508 100644 --- a/src/leveldbwrapper.h +++ b/src/dbwrapper.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_LEVELDBWRAPPER_H -#define BITCOIN_LEVELDBWRAPPER_H +#ifndef BITCOIN_DBWRAPPER_H +#define BITCOIN_DBWRAPPER_H #include "clientversion.h" #include "serialize.h" @@ -17,18 +17,18 @@ #include <leveldb/db.h> #include <leveldb/write_batch.h> -class leveldb_error : public std::runtime_error +class dbwrapper_error : public std::runtime_error { public: - leveldb_error(const std::string& msg) : std::runtime_error(msg) {} + dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} }; -void HandleError(const leveldb::Status& status) throw(leveldb_error); +void HandleError(const leveldb::Status& status) throw(dbwrapper_error); -/** Batch of changes queued to be written to a CLevelDBWrapper */ -class CLevelDBBatch +/** Batch of changes queued to be written to a CDBWrapper */ +class CDBBatch { - friend class CLevelDBWrapper; + friend class CDBWrapper; private: leveldb::WriteBatch batch; @@ -38,7 +38,7 @@ public: /** * @param[in] obfuscate_key If passed, XOR data with this key. */ - CLevelDBBatch(const std::vector<unsigned char> *obfuscate_key) : obfuscate_key(obfuscate_key) { }; + CDBBatch(const std::vector<unsigned char> *obfuscate_key) : obfuscate_key(obfuscate_key) { }; template <typename K, typename V> void Write(const K& key, const V& value) @@ -68,8 +68,8 @@ public: batch.Delete(slKey); } }; - -class CLevelDBIterator + +class CDBIterator { private: leveldb::Iterator *piter; @@ -81,14 +81,13 @@ public: * @param[in] piterIn The original leveldb iterator. * @param[in] obfuscate_key If passed, XOR data with this key. */ - CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector<unsigned char>* obfuscate_key) : + CDBIterator(leveldb::Iterator *piterIn, const std::vector<unsigned char>* obfuscate_key) : piter(piterIn), obfuscate_key(obfuscate_key) { }; - ~CLevelDBIterator(); + ~CDBIterator(); bool Valid(); void SeekToFirst(); - void SeekToLast(); template<typename K> void Seek(const K& key) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); @@ -99,7 +98,6 @@ public: } void Next(); - void Prev(); template<typename K> bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); @@ -133,8 +131,8 @@ public: } }; - -class CLevelDBWrapper + +class CDBWrapper { private: //! custom environment this database is using (may be NULL in case of default environment) @@ -163,10 +161,10 @@ private: //! the key under which the obfuscation key is stored static const std::string OBFUSCATE_KEY_KEY; - + //! the length of the obfuscate key in number of bytes static const unsigned int OBFUSCATE_KEY_NUM_BYTES; - + std::vector<unsigned char> CreateObfuscateKey() const; public: @@ -178,11 +176,11 @@ public: * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR * with a zero'd byte array. */ - CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); - ~CLevelDBWrapper(); + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + ~CDBWrapper(); template <typename K, typename V> - bool Read(const K& key, V& value) const throw(leveldb_error) + bool Read(const K& key, V& value) const throw(dbwrapper_error) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -208,15 +206,15 @@ public: } template <typename K, typename V> - bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error) + bool Write(const K& key, const V& value, bool fSync = false) throw(dbwrapper_error) { - CLevelDBBatch batch(&obfuscate_key); + CDBBatch batch(&obfuscate_key); batch.Write(key, value); return WriteBatch(batch, fSync); } template <typename K> - bool Exists(const K& key) const throw(leveldb_error) + bool Exists(const K& key) const throw(dbwrapper_error) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -235,14 +233,14 @@ public: } template <typename K> - bool Erase(const K& key, bool fSync = false) throw(leveldb_error) + bool Erase(const K& key, bool fSync = false) throw(dbwrapper_error) { - CLevelDBBatch batch(&obfuscate_key); + CDBBatch batch(&obfuscate_key); batch.Erase(key); return WriteBatch(batch, fSync); } - bool WriteBatch(CLevelDBBatch& batch, bool fSync = false) throw(leveldb_error); + bool WriteBatch(CDBBatch& batch, bool fSync = false) throw(dbwrapper_error); // not available for LevelDB; provide for compatibility with BDB bool Flush() @@ -250,15 +248,15 @@ public: return true; } - bool Sync() throw(leveldb_error) + bool Sync() throw(dbwrapper_error) { - CLevelDBBatch batch(&obfuscate_key); + CDBBatch batch(&obfuscate_key); return WriteBatch(batch, true); } - CLevelDBIterator *NewIterator() + CDBIterator *NewIterator() { - return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); + return new CDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); } /** @@ -278,5 +276,5 @@ public: }; -#endif // BITCOIN_LEVELDBWRAPPER_H +#endif // BITCOIN_DBWRAPPER_H diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 0a7f903e9f..8698abb900 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -38,6 +38,9 @@ #include <boost/foreach.hpp> #include <boost/scoped_ptr.hpp> +/** Maximum size of http request (request line + headers) */ +static const size_t MAX_HEADERS_SIZE = 8192; + /** HTTP request work item */ class HTTPWorkItem : public HTTPClosure { @@ -414,6 +417,7 @@ bool InitHTTPServer() } evhttp_set_timeout(http, GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); + evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, NULL); diff --git a/src/init.cpp b/src/init.cpp index d899a1cf98..4189907b9f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -320,6 +320,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-dbcache=<n>", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file") + " " + _("on startup")); strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); + strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 @@ -367,6 +369,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-whitebind=<addr>", _("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt("-whitelist=<netmask>", _("Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway")); + strUsage += HelpMessageOpt("-maxuploadtarget=<n>", strprintf(_("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)"), 0)); #ifdef ENABLE_WALLET strUsage += HelpMessageGroup(_("Wallet options:")); @@ -428,6 +431,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %u)"), 1)); if (showDebug) { + strUsage += HelpMessageOpt("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS)); strUsage += HelpMessageOpt("-limitfreerelay=<n>", strprintf("Continuously rate-limit free transactions to <n>*1000 bytes per minute (default: %u)", 15)); strUsage += HelpMessageOpt("-relaypriority", strprintf("Require high priority for relaying free or low-fee transactions (default: %u)", 1)); strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf("Limit size of signature cache to <n> entries (default: %u)", 50000)); @@ -725,6 +729,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // Set this early so that parameter interactions go to console fPrintToConsole = GetBoolArg("-printtoconsole", false); fLogTimestamps = GetBoolArg("-logtimestamps", true); + fLogTimeMicros = GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); fLogIPs = GetBoolArg("-logips", false); LogPrintf("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"); @@ -843,6 +848,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) fCheckBlockIndex = GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); fCheckpointsEnabled = GetBoolArg("-checkpoints", true); + // -mempoollimit limits + int64_t nMempoolSizeLimit = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolDescendantSizeLimit = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + if (nMempoolSizeLimit < 0 || nMempoolSizeLimit < nMempoolDescendantSizeLimit * 40) + return InitError(strprintf(_("Error: -maxmempool must be at least %d MB"), GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) / 25)); + // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency nScriptCheckThreads = GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (nScriptCheckThreads <= 0) @@ -1169,6 +1180,9 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) RegisterValidationInterface(pzmqNotificationInterface); } #endif + if (mapArgs.count("-maxuploadtarget")) { + CNode::SetMaxOutboundTarget(GetArg("-maxuploadtarget", 0)*1024*1024); + } // ********************************************************* Step 7: load block chain @@ -1500,10 +1514,10 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // if pruning, unset the service bit and perform the initial blockstore prune // after any wallet rescanning has taken place. if (fPruneMode) { - uiInterface.InitMessage(_("Pruning blockstore...")); LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices &= ~NODE_NETWORK; if (!fReindex) { + uiInterface.InitMessage(_("Pruning blockstore...")); PruneAndFlush(); } } diff --git a/src/main.cpp b/src/main.cpp index e931d40c94..e038fe3663 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -75,7 +75,7 @@ uint64_t nPruneTarget = 0; bool fAlerts = DEFAULT_ALERTS; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ -CFeeRate minRelayTxFee = CFeeRate(5000); +CFeeRate minRelayTxFee = CFeeRate(1000); CTxMemPool mempool(::minRelayTxFee); @@ -650,10 +650,35 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) return true; } -bool CheckFinalTx(const CTransaction &tx) +bool CheckFinalTx(const CTransaction &tx, int flags) { AssertLockHeld(cs_main); - return IsFinalTx(tx, chainActive.Height() + 1, GetAdjustedTime()); + + // By convention a negative value for flags indicates that the + // current network-enforced consensus rules should be used. In + // a future soft-fork scenario that would mean checking which + // rules would be enforced for the next block and setting the + // appropriate flags. At the present time no soft-forks are + // scheduled, so no flags are set. + flags = std::max(flags, 0); + + // CheckFinalTx() uses chainActive.Height()+1 to evaluate + // nLockTime because when IsFinalTx() is called within + // CBlock::AcceptBlock(), the height of the block *being* + // evaluated is what is used. Thus if we want to know if a + // transaction can be part of the *next* block, we need to call + // IsFinalTx() with one more than chainActive.Height(). + const int nBlockHeight = chainActive.Height() + 1; + + // Timestamps on the other hand don't get any special treatment, + // because we can't know what timestamp the next block will have, + // and there aren't timestamp applications where it matters. + // However this changes once median past time-locks are enforced: + const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) + ? chainActive.Tip()->GetMedianTimePast() + : GetAdjustedTime(); + + return IsFinalTx(tx, nBlockHeight, nBlockTime); } unsigned int GetLegacySigOpCount(const CTransaction& tx) @@ -740,17 +765,14 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) return true; } -CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree) +CAmount GetMinRelayFee(const CTransaction& tx, const CTxMemPool& pool, unsigned int nBytes, bool fAllowFree) { - { - LOCK(mempool.cs); - uint256 hash = tx.GetHash(); - double dPriorityDelta = 0; - CAmount nFeeDelta = 0; - mempool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); - if (dPriorityDelta > 0 || nFeeDelta > 0) - return 0; - } + uint256 hash = tx.GetHash(); + double dPriorityDelta = 0; + CAmount nFeeDelta = 0; + pool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); + if (dPriorityDelta > 0 || nFeeDelta > 0) + return 0; CAmount nMinFee = ::minRelayTxFee.GetFee(nBytes); @@ -779,7 +801,7 @@ static std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectAbsurdFee) + bool* pfMissingInputs, bool fOverrideMempoolLimit, bool fRejectAbsurdFee) { AssertLockHeld(cs_main); if (pfMissingInputs) @@ -800,7 +822,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. - if (!CheckFinalTx(tx)) + if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.DoS(0, false, REJECT_NONSTANDARD, "non-final"); // is it already in the memory pool? @@ -879,17 +901,20 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CAmount nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx)); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx)); unsigned int nSize = entry.GetTxSize(); // Don't accept it if it can't get into a block - CAmount txMinFee = GetMinRelayFee(tx, nSize, true); + CAmount txMinFee = GetMinRelayFee(tx, pool, nSize, true); if (fLimitFree && nFees < txMinFee) return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient fee", false, strprintf("%d < %d", nFees, txMinFee)); - // Require that free transactions have sufficient priority to be mined in the next block. - if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) { + CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); + if (mempoolRejectFee > 0 && nFees < mempoolRejectFee) { + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); + } else if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) { + // Require that free transactions have sufficient priority to be mined in the next block. return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); } @@ -954,6 +979,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + + // trim mempool and check if tx was trimmed + if (!fOverrideMempoolLimit) { + int expired = pool.Expire(GetTime() - GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + if (expired != 0) + LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); + + pool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (!pool.exists(tx.GetHash())) + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); + } } SyncWithWallets(tx, NULL); @@ -1740,11 +1776,18 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; - // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, when 75% of the network has upgraded: + // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, + // when 75% of the network has upgraded: if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { flags |= SCRIPT_VERIFY_DERSIG; } + // Start enforcing CHECKLOCKTIMEVERIFY, (BIP65) for block.nVersion=4 + // blocks, when 75% of the network has upgraded: + if (block.nVersion >= 4 && IsSuperMajority(4, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; + } + CBlockUndo blockundo; CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); @@ -1881,7 +1924,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { std::set<int> setFilesToPrune; bool fFlushForPrune = false; try { - if (fPruneMode && fCheckForPruning) { + if (fPruneMode && fCheckForPruning && !fReindex) { FindFilesToPrune(setFilesToPrune); fCheckForPruning = false; if (!setFilesToPrune.empty()) { @@ -2020,7 +2063,7 @@ void static UpdateTip(CBlockIndex *pindexNew) { } } -/** Disconnect chainActive's tip. */ +/** Disconnect chainActive's tip. You want to manually re-limit mempool size after this */ bool static DisconnectTip(CValidationState &state) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2047,7 +2090,7 @@ bool static DisconnectTip(CValidationState &state) { // ignore validation errors in resurrected transactions list<CTransaction> removed; CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) { + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { mempool.remove(tx, removed, true); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); @@ -2220,9 +2263,11 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. + bool fBlocksDisconnected = false; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { if (!DisconnectTip(state)) return false; + fBlocksDisconnected = true; } // Build list of new blocks to connect. @@ -2268,6 +2313,9 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo } } + if (fBlocksDisconnected) + mempool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + // Callbacks/notifications for a new best chain. if (fInvalidFound) CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); @@ -2354,6 +2402,8 @@ bool InvalidateBlock(CValidationState& state, CBlockIndex *pindex) { } } + mempool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. BlockMap::iterator it = mapBlockIndex.begin(); @@ -2684,6 +2734,11 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta return state.Invalid(error("%s : rejected nVersion=2 block", __func__), REJECT_OBSOLETE, "bad-version"); + // Reject block.nVersion=3 blocks when 95% (75% on testnet) of the network has upgraded: + if (block.nVersion < 4 && IsSuperMajority(4, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) + return state.Invalid(error("%s : rejected nVersion=3 block", __func__), + REJECT_OBSOLETE, "bad-version"); + return true; } @@ -2693,10 +2748,15 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn const Consensus::Params& consensusParams = Params().GetConsensus(); // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) - if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) { + BOOST_FOREACH(const CTransaction& tx, block.vtx) { + int nLockTimeFlags = 0; + int64_t nLockTimeCutoff = (nLockTimeFlags & LOCKTIME_MEDIAN_TIME_PAST) + ? pindexPrev->GetMedianTimePast() + : block.GetBlockTime(); + if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { return state.DoS(10, error("%s: contains a non-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); } + } // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): @@ -3775,6 +3835,16 @@ void static ProcessGetData(CNode* pfrom) } } } + // disconnect node in case we have reached the outbound limit for serving historical blocks + static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical + if (send && CNode::OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) ) + { + LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + + //disconnect node + pfrom->fDisconnect = true; + send = false; + } // Pruned nodes may have deleted the block, so check whether // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) @@ -4290,10 +4360,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, RelayTransaction(tx); vWorkQueue.push_back(inv.hash); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u)\n", + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom->id, tx.GetHash().ToString(), - mempool.size()); + mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; diff --git a/src/main.h b/src/main.h index a6001eed8f..65732d770f 100644 --- a/src/main.h +++ b/src/main.h @@ -51,6 +51,10 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500; +/** Default for -maxmempool, maximum megabytes of mempool memory usage */ +static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300; +/** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ +static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 72; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -225,7 +229,7 @@ void PruneAndFlush(); /** (try to) add transaction to memory pool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectAbsurdFee=false); + bool* pfMissingInputs, bool fOverrideMempoolLimit=false, bool fRejectAbsurdFee=false); struct CNodeStateStats { @@ -304,8 +308,10 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); * Check if transaction will be final in the next block to be created. * * Calls IsFinalTx() with current block height and appropriate block time. + * + * See consensus/consensus.h for flag definitions. */ -bool CheckFinalTx(const CTransaction &tx); +bool CheckFinalTx(const CTransaction &tx, int flags = -1); /** * Closure representing one script verification diff --git a/src/miner.cpp b/src/miner.cpp index 42c8bb970b..053d9cdbc4 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -148,6 +148,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) CBlockIndex* pindexPrev = chainActive.Tip(); const int nHeight = pindexPrev->nHeight + 1; pblock->nTime = GetAdjustedTime(); + const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); CCoinsViewCache view(pcoinsTip); // Priority order to process transactions @@ -162,7 +163,12 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) mi != mempool.mapTx.end(); ++mi) { const CTransaction& tx = mi->GetTx(); - if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight, pblock->nTime)) + + int64_t nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST) + ? nMedianTimePast + : pblock->GetBlockTime(); + + if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight, nLockTimeCutoff)) continue; COrphan* porphan = NULL; diff --git a/src/net.cpp b/src/net.cpp index 87c4f0af0a..e18e8d0e29 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -12,6 +12,7 @@ #include "addrman.h" #include "chainparams.h" #include "clientversion.h" +#include "consensus/consensus.h" #include "crypto/common.h" #include "hash.h" #include "primitives/transaction.h" @@ -326,6 +327,11 @@ uint64_t CNode::nTotalBytesSent = 0; CCriticalSection CNode::cs_totalBytesRecv; CCriticalSection CNode::cs_totalBytesSent; +uint64_t CNode::nMaxOutboundLimit = 0; +uint64_t CNode::nMaxOutboundTotalBytesSentInCycle = 0; +uint64_t CNode::nMaxOutboundTimeframe = 60*60*24; //1 day +uint64_t CNode::nMaxOutboundCycleStartTime = 0; + CNode* FindNode(const CNetAddr& ip) { LOCK(cs_vNodes); @@ -963,6 +969,15 @@ static void AcceptConnection(const ListenSocket& hListenSocket) { return; } + // According to the internet TCP_NODELAY is not carried into accepted sockets + // on all platforms. Set it again here just to be sure. + int set = 1; +#ifdef WIN32 + setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); +#else + setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); +#endif + if (CNode::IsBanned(addr) && !whitelisted) { LogPrintf("connection from %s dropped (banned)\n", addr.ToString()); @@ -1790,8 +1805,11 @@ bool BindListenPort(const CService &addrBind, string& strError, bool fWhiteliste // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (void*)&nOne, sizeof(int)); + // Disable Nagle's algorithm + setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&nOne, sizeof(int)); #else setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (const char*)&nOne, sizeof(int)); + setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&nOne, sizeof(int)); #endif // Set to non-blocking, incoming connections will also inherit this @@ -2071,6 +2089,94 @@ void CNode::RecordBytesSent(uint64_t bytes) { LOCK(cs_totalBytesSent); nTotalBytesSent += bytes; + + uint64_t now = GetTime(); + if (nMaxOutboundCycleStartTime + nMaxOutboundTimeframe < now) + { + // timeframe expired, reset cycle + nMaxOutboundCycleStartTime = now; + nMaxOutboundTotalBytesSentInCycle = 0; + } + + // TODO, exclude whitebind peers + nMaxOutboundTotalBytesSentInCycle += bytes; +} + +void CNode::SetMaxOutboundTarget(uint64_t limit) +{ + LOCK(cs_totalBytesSent); + uint64_t recommendedMinimum = (nMaxOutboundTimeframe / 600) * MAX_BLOCK_SIZE; + nMaxOutboundLimit = limit; + + if (limit < recommendedMinimum) + LogPrintf("Max outbound target is very small (%s) and will be overshot. Recommended minimum is %s\n.", nMaxOutboundLimit, recommendedMinimum); +} + +uint64_t CNode::GetMaxOutboundTarget() +{ + LOCK(cs_totalBytesSent); + return nMaxOutboundLimit; +} + +uint64_t CNode::GetMaxOutboundTimeframe() +{ + LOCK(cs_totalBytesSent); + return nMaxOutboundTimeframe; +} + +uint64_t CNode::GetMaxOutboundTimeLeftInCycle() +{ + LOCK(cs_totalBytesSent); + if (nMaxOutboundLimit == 0) + return 0; + + if (nMaxOutboundCycleStartTime == 0) + return nMaxOutboundTimeframe; + + uint64_t cycleEndTime = nMaxOutboundCycleStartTime + nMaxOutboundTimeframe; + uint64_t now = GetTime(); + return (cycleEndTime < now) ? 0 : cycleEndTime - GetTime(); +} + +void CNode::SetMaxOutboundTimeframe(uint64_t timeframe) +{ + LOCK(cs_totalBytesSent); + if (nMaxOutboundTimeframe != timeframe) + { + // reset measure-cycle in case of changing + // the timeframe + nMaxOutboundCycleStartTime = GetTime(); + } + nMaxOutboundTimeframe = timeframe; +} + +bool CNode::OutboundTargetReached(bool historicalBlockServingLimit) +{ + LOCK(cs_totalBytesSent); + if (nMaxOutboundLimit == 0) + return false; + + if (historicalBlockServingLimit) + { + // keep a large enought buffer to at least relay each block once + uint64_t timeLeftInCycle = GetMaxOutboundTimeLeftInCycle(); + uint64_t buffer = timeLeftInCycle / 600 * MAX_BLOCK_SIZE; + if (buffer >= nMaxOutboundLimit || nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit - buffer) + return true; + } + else if (nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit) + return true; + + return false; +} + +uint64_t CNode::GetOutboundTargetBytesLeft() +{ + LOCK(cs_totalBytesSent); + if (nMaxOutboundLimit == 0) + return 0; + + return (nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit) ? 0 : nMaxOutboundLimit - nMaxOutboundTotalBytesSentInCycle; } uint64_t CNode::GetTotalBytesRecv() @@ -400,6 +400,12 @@ private: static uint64_t nTotalBytesRecv; static uint64_t nTotalBytesSent; + // outbound limit & stats + static uint64_t nMaxOutboundTotalBytesSentInCycle; + static uint64_t nMaxOutboundCycleStartTime; + static uint64_t nMaxOutboundLimit; + static uint64_t nMaxOutboundTimeframe; + CNode(const CNode&); void operator=(const CNode&); @@ -701,6 +707,27 @@ public: static uint64_t GetTotalBytesRecv(); static uint64_t GetTotalBytesSent(); + + //!set the max outbound target in bytes + static void SetMaxOutboundTarget(uint64_t limit); + static uint64_t GetMaxOutboundTarget(); + + //!set the timeframe for the max outbound target + static void SetMaxOutboundTimeframe(uint64_t timeframe); + static uint64_t GetMaxOutboundTimeframe(); + + //!check if the outbound target is reached + // if param historicalBlockServingLimit is set true, the function will + // response true if the limit for serving historical blocks has been reached + static bool OutboundTargetReached(bool historicalBlockServingLimit); + + //!response the bytes left in the current max outbound cycle + // in case of no limit, it will always response 0 + static uint64_t GetOutboundTargetBytesLeft(); + + //!response the time in second left in the current max outbound cycle + // in case of no limit, it will always response 0 + static uint64_t GetMaxOutboundTimeLeftInCycle(); }; diff --git a/src/netbase.cpp b/src/netbase.cpp index c3d56d9184..f5316965ce 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -444,12 +444,19 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe if (hSocket == INVALID_SOCKET) return false; -#ifdef SO_NOSIGPIPE int set = 1; +#ifdef SO_NOSIGPIPE // Different way of disabling SIGPIPE on BSD setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); #endif + //Disable Nagle's algorithm +#ifdef WIN32 + setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); +#else + setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); +#endif + // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) return error("ConnectSocketDirectly: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); diff --git a/src/policy/policy.h b/src/policy/policy.h index 0ea0d435ad..7027f1402f 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -43,6 +43,9 @@ static const unsigned int STANDARD_SCRIPT_VERIFY_FLAGS = MANDATORY_SCRIPT_VERIFY /** For convenience, standard but not mandatory verify flags. */ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_VERIFY_FLAGS & ~MANDATORY_SCRIPT_VERIFY_FLAGS; +/** Used as the flags parameter to CheckFinalTx() in non-consensus code */ +static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_MEDIAN_TIME_PAST; + bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); /** * Check for standard transaction types diff --git a/src/primitives/block.h b/src/primitives/block.h index 86106098f5..54731ff557 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -21,7 +21,7 @@ class CBlockHeader { public: // header - static const int32_t CURRENT_VERSION=3; + static const int32_t CURRENT_VERSION=4; int32_t nVersion; uint256 hashPrevBlock; uint256 hashMerkleRoot; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 89d2d44fd7..bda8acff15 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -563,7 +563,7 @@ int main(int argc, char *argv[]) // Show help message immediately after parsing command-line options (for "-lang") and setting locale, // but before showing splash screen. - if (mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) + if (mapArgs.count("-?") || mapArgs.count("-h") || mapArgs.count("-help") || mapArgs.count("-version")) { HelpMessageDialog help(NULL, mapArgs.count("-version")); help.showOrPrint(); diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 545ac12890..4786d72a3f 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -648,6 +648,7 @@ UniValue getblockchaininfo(const UniValue& params, bool fHelp) UniValue softforks(UniValue::VARR); softforks.push_back(SoftForkDesc("bip34", 2, tip, consensusParams)); softforks.push_back(SoftForkDesc("bip66", 3, tip, consensusParams)); + softforks.push_back(SoftForkDesc("bip65", 4, tip, consensusParams)); obj.push_back(Pair("softforks", softforks)); if (fPruneMode) @@ -772,6 +773,9 @@ UniValue mempoolInfoToJSON() ret.push_back(Pair("size", (int64_t) mempool.size())); ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); + size_t maxmempool = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + ret.push_back(Pair("maxmempool", (int64_t) maxmempool)); + ret.push_back(Pair("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK()))); return ret; } diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 4064c2fee3..343b6234d4 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -76,6 +76,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawtransaction", 1 }, { "createrawtransaction", 0 }, { "createrawtransaction", 1 }, + { "createrawtransaction", 2 }, { "signrawtransaction", 1 }, { "signrawtransaction", 2 }, { "sendrawtransaction", 1 }, diff --git a/src/rpcnet.cpp b/src/rpcnet.cpp index 7746be25f7..6b4815ebd8 100644 --- a/src/rpcnet.cpp +++ b/src/rpcnet.cpp @@ -379,6 +379,15 @@ UniValue getnettotals(const UniValue& params, bool fHelp) obj.push_back(Pair("totalbytesrecv", CNode::GetTotalBytesRecv())); obj.push_back(Pair("totalbytessent", CNode::GetTotalBytesSent())); obj.push_back(Pair("timemillis", GetTimeMillis())); + + UniValue outboundLimit(UniValue::VOBJ); + outboundLimit.push_back(Pair("timeframe", CNode::GetMaxOutboundTimeframe())); + outboundLimit.push_back(Pair("target", CNode::GetMaxOutboundTarget())); + outboundLimit.push_back(Pair("target_reached", CNode::OutboundTargetReached(false))); + outboundLimit.push_back(Pair("serve_historical_blocks", !CNode::OutboundTargetReached(true))); + outboundLimit.push_back(Pair("bytes_left_in_cycle", CNode::GetOutboundTargetBytesLeft())); + outboundLimit.push_back(Pair("time_left_in_cycle", CNode::GetMaxOutboundTimeLeftInCycle())); + obj.push_back(Pair("uploadtarget", outboundLimit)); return obj; } diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 4dec53396d..5f3363d097 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -316,9 +316,9 @@ UniValue verifytxoutproof(const UniValue& params, bool fHelp) UniValue createrawtransaction(const UniValue& params, bool fHelp) { - if (fHelp || params.size() != 2) + if (fHelp || params.size() < 2 || params.size() > 3) throw runtime_error( - "createrawtransaction [{\"txid\":\"id\",\"vout\":n},...] {\"address\":amount,\"data\":\"hex\",...}\n" + "createrawtransaction [{\"txid\":\"id\",\"vout\":n},...] {\"address\":amount,\"data\":\"hex\",...} ( locktime )\n" "\nCreate a transaction spending the given inputs and creating new outputs.\n" "Outputs can be addresses or data.\n" "Returns hex-encoded raw transaction.\n" @@ -340,6 +340,7 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) " \"data\": \"hex\", (string, required) The key is \"data\", the value is hex encoded data\n" " ...\n" " }\n" + "3. locktime (numeric, optional, default=0) Raw locktime. Non-0 value also locktime-activates inputs\n" "\nResult:\n" "\"transaction\" (string) hex string of the transaction\n" @@ -351,13 +352,22 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) ); LOCK(cs_main); - RPCTypeCheck(params, boost::assign::list_of(UniValue::VARR)(UniValue::VOBJ)); + RPCTypeCheck(params, boost::assign::list_of(UniValue::VARR)(UniValue::VOBJ)(UniValue::VNUM), true); + if (params[0].isNull() || params[1].isNull()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); UniValue inputs = params[0].get_array(); UniValue sendTo = params[1].get_obj(); CMutableTransaction rawTx; + if (params.size() > 2 && !params[2].isNull()) { + int64_t nLockTime = params[2].get_int64(); + if (nLockTime < 0 || nLockTime > std::numeric_limits<uint32_t>::max()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, locktime out of range"); + rawTx.nLockTime = nLockTime; + } + for (unsigned int idx = 0; idx < inputs.size(); idx++) { const UniValue& input = inputs[idx]; const UniValue& o = input.get_obj(); @@ -371,7 +381,9 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) if (nOutput < 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); - CTxIn in(COutPoint(txid, nOutput)); + uint32_t nSequence = (rawTx.nLockTime ? std::numeric_limits<uint32_t>::max() - 1 : std::numeric_limits<uint32_t>::max()); + CTxIn in(COutPoint(txid, nOutput), CScript(), nSequence); + rawTx.vin.push_back(in); } @@ -809,7 +821,7 @@ UniValue sendrawtransaction(const UniValue& params, bool fHelp) // push to local node and sync with wallets CValidationState state; bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, !fOverrideFees)) { + if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, false, !fOverrideFees)) { if (state.IsInvalid()) { throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); } else { diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 0320577797..a48ff1e18d 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -44,9 +44,10 @@ typedef enum bitcoinconsensus_error_t /** Script verification flags */ enum { - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE = 0, - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH = (1U << 0), // evaluate P2SH (BIP16) subscripts - bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG = (1U << 2), // enforce strict DER (BIP66) compliance + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_NONE = 0, + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH = (1U << 0), // evaluate P2SH (BIP16) subscripts + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG = (1U << 2), // enforce strict DER (BIP66) compliance + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) }; /// Returns 1 if the input nIn of the serialized transaction pointed to by diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index d3aec26020..6a20d497c0 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -273,7 +273,7 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un return set_error(serror, SCRIPT_ERR_PUSH_SIZE); // Note how OP_RESERVED does not count towards the opcode limit. - if (opcode > OP_16 && ++nOpCount > 201) + if (opcode > OP_16 && ++nOpCount > MAX_OPS_PER_SCRIPT) return set_error(serror, SCRIPT_ERR_OP_COUNT); if (opcode == OP_CAT || @@ -869,10 +869,10 @@ bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, un return set_error(serror, SCRIPT_ERR_INVALID_STACK_OPERATION); int nKeysCount = CScriptNum(stacktop(-i), fRequireMinimal).getint(); - if (nKeysCount < 0 || nKeysCount > 20) + if (nKeysCount < 0 || nKeysCount > MAX_PUBKEYS_PER_MULTISIG) return set_error(serror, SCRIPT_ERR_PUBKEY_COUNT); nOpCount += nKeysCount; - if (nOpCount > 201) + if (nOpCount > MAX_OPS_PER_SCRIPT) return set_error(serror, SCRIPT_ERR_OP_COUNT); int ikey = ++i; i += nKeysCount; diff --git a/src/script/script.cpp b/src/script/script.cpp index 9a0c067a33..263c89defe 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -170,7 +170,7 @@ unsigned int CScript::GetSigOpCount(bool fAccurate) const if (fAccurate && lastOpcode >= OP_1 && lastOpcode <= OP_16) n += DecodeOP_N(lastOpcode); else - n += 20; + n += MAX_PUBKEYS_PER_MULTISIG; } lastOpcode = opcode; } diff --git a/src/script/script.h b/src/script/script.h index cdc9a71bb2..a38d33a189 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -17,7 +17,14 @@ #include <string> #include <vector> -static const unsigned int MAX_SCRIPT_ELEMENT_SIZE = 520; // bytes +// Maximum number of bytes pushable to the stack +static const unsigned int MAX_SCRIPT_ELEMENT_SIZE = 520; + +// Maximum number of non-push operations per script +static const int MAX_OPS_PER_SCRIPT = 201; + +// Maximum number of public keys per multisig +static const int MAX_PUBKEYS_PER_MULTISIG = 20; // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp index dd3c51d09b..468eda1c9b 100644 --- a/src/test/alert_tests.cpp +++ b/src/test/alert_tests.cpp @@ -217,10 +217,12 @@ BOOST_AUTO_TEST_CASE(PartitionAlert) // use them } + strMiscWarning = ""; + // Test 1: chain with blocks every nPowTargetSpacing seconds, // as normal, no worries: PartitionCheck(falseFunc, csDummy, &indexDummy[99], nPowTargetSpacing); - BOOST_CHECK(strMiscWarning.empty()); + BOOST_CHECK_MESSAGE(strMiscWarning.empty(), strMiscWarning); // Test 2: go 3.5 hours without a block, expect a warning: now += 3*60*60+30*60; diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index 5cad5af7c3..d3c8594342 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -193,5 +193,9 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xc5b93064159b3b2d6ab506a41b1f50463771b988 EQUAL"]], "0100000001000100000000000000000000000000000000000000000000000000000000000000000000030251b1000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +["A transaction with a non-standard DER signature."], +[[["b1dbc81696c8a9c0fccd0693ab66d7c368dbc38c0def4e800685560ddd1b2132", 0, "DUP HASH160 0x14 0x4b3bd7eba3bc0284fd3007be7f3be275e94f5826 EQUALVERIFY CHECKSIG"]], +"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "P2SH,DERSIG"], + ["Make diffs cleaner by leaving a comment here without comma at the end"] ] diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index 9744a3c848..0dfef73ae5 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -229,5 +229,9 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xc5b93064159b3b2d6ab506a41b1f50463771b988 EQUAL"]], "0100000001000100000000000000000000000000000000000000000000000000000000000000000000030251b1000000000100000000000000000001000000", "P2SH,CHECKLOCKTIMEVERIFY"], +["A transaction with a non-standard DER signature."], +[[["b1dbc81696c8a9c0fccd0693ab66d7c368dbc38c0def4e800685560ddd1b2132", 0, "DUP HASH160 0x14 0x4b3bd7eba3bc0284fd3007be7f3be275e94f5826 EQUALVERIFY CHECKSIG"]], +"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "P2SH"], + ["Make diffs cleaner by leaving a comment here without comma at the end"] ] diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 606313b004..8b6b0697ab 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include "uint256.h" #include "random.h" #include "test/test_bitcoin.h" @@ -25,15 +25,15 @@ bool is_null_key(const vector<unsigned char>& key) { return isnull; } -BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) -BOOST_AUTO_TEST_CASE(leveldbwrapper) +BOOST_AUTO_TEST_CASE(dbwrapper) { // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'k'; uint256 in = GetRandHash(); uint256 res; @@ -48,13 +48,13 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper) } // Test batch operations -BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) +BOOST_AUTO_TEST_CASE(dbwrapper_batch) { // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'i'; uint256 in = GetRandHash(); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CLevelDBBatch batch(&dbw.GetObfuscateKey()); + CDBBatch batch(&dbw.GetObfuscateKey()); batch.Write(key, in); batch.Write(key2, in2); @@ -85,13 +85,13 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) } } -BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) +BOOST_AUTO_TEST_CASE(dbwrapper_iterator) { // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); + CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); // The two keys are intentionally chosen for ordering char key = 'j'; @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) uint256 in2 = GetRandHash(); BOOST_CHECK(dbw.Write(key2, in2)); - boost::scoped_ptr<CLevelDBIterator> it(const_cast<CLevelDBWrapper*>(&dbw)->NewIterator()); + boost::scoped_ptr<CDBIterator> it(const_cast<CDBWrapper*>(&dbw)->NewIterator()); // Be sure to seek past the obfuscation key (if it exists) it->Seek(key); @@ -134,7 +134,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = GetRandHash(); uint256 res; @@ -147,7 +147,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) delete dbw; // Now, set up another wrapper that wants to obfuscate the same directory - CLevelDBWrapper odbw(ph, (1 << 10), false, false, true); + CDBWrapper odbw(ph, (1 << 10), false, false, true); // Check that the key/val we wrote with unobfuscated wrapper exists and // is readable. @@ -175,7 +175,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false); + CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = GetRandHash(); uint256 res; @@ -188,7 +188,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) delete dbw; // Simulate a -reindex by wiping the existing data store - CLevelDBWrapper odbw(ph, (1 << 10), false, true, true); + CDBWrapper odbw(ph, (1 << 10), false, true, true); // Check that the key/val we wrote with unobfuscated wrapper doesn't exist uint256 res2; diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 5bf1e98e8f..0cf906a259 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -153,11 +153,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> sortedOrder; sortedOrder.resize(5); - sortedOrder[0] = tx2.GetHash().ToString(); // 20000 - sortedOrder[1] = tx4.GetHash().ToString(); // 15000 + sortedOrder[0] = tx3.GetHash().ToString(); // 0 + sortedOrder[1] = tx5.GetHash().ToString(); // 10000 sortedOrder[2] = tx1.GetHash().ToString(); // 10000 - sortedOrder[3] = tx5.GetHash().ToString(); // 10000 - sortedOrder[4] = tx3.GetHash().ToString(); // 0 + sortedOrder[3] = tx4.GetHash().ToString(); // 15000 + sortedOrder[4] = tx2.GetHash().ToString(); // 20000 CheckSort(pool, sortedOrder); /* low fee but with high fee child */ @@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.addUnchecked(tx6.GetHash(), CTxMemPoolEntry(tx6, 0LL, 1, 10.0, 1, true)); BOOST_CHECK_EQUAL(pool.size(), 6); // Check that at this point, tx6 is sorted low - sortedOrder.push_back(tx6.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); CheckSort(pool, sortedOrder); CTxMemPool::setEntries setAncestors; @@ -194,9 +194,9 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 7); // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... - sortedOrder.erase(sortedOrder.end()-1); - sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); - sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); + sortedOrder.erase(sortedOrder.begin()); + sortedOrder.push_back(tx6.GetHash().ToString()); + sortedOrder.push_back(tx7.GetHash().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.addUnchecked(tx8.GetHash(), CTxMemPoolEntry(tx8, 0LL, 2, 10.0, 1, true), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high - sortedOrder.push_back(tx8.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx8.GetHash().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) // tx9 should be sorted low BOOST_CHECK_EQUAL(pool.size(), 9); - sortedOrder.push_back(tx9.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx9.GetHash().ToString()); CheckSort(pool, sortedOrder); std::vector<std::string> snapshotOrder = sortedOrder; @@ -255,21 +255,21 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) * tx8 and tx9 should both now be sorted higher * Final order after tx10 is added: * - * tx7 = 2.2M (4 txs) - * tx6 = 2.2M (5 txs) - * tx10 = 200k (1 tx) - * tx8 = 200k (2 txs) - * tx9 = 200k (2 txs) - * tx2 = 20000 (1) - * tx4 = 15000 (1) - * tx1 = 10000 (1) - * tx5 = 10000 (1) * tx3 = 0 (1) + * tx5 = 10000 (1) + * tx1 = 10000 (1) + * tx4 = 15000 (1) + * tx2 = 20000 (1) + * tx9 = 200k (2 txs) + * tx8 = 200k (2 txs) + * tx10 = 200k (1 tx) + * tx6 = 2.2M (5 txs) + * tx7 = 2.2M (4 txs) */ - sortedOrder.erase(sortedOrder.end()-2, sortedOrder.end()); // take out tx8, tx9 from the end - sortedOrder.insert(sortedOrder.begin()+2, tx10.GetHash().ToString()); // tx10 is after tx6 - sortedOrder.insert(sortedOrder.begin()+3, tx9.GetHash().ToString()); - sortedOrder.insert(sortedOrder.begin()+3, tx8.GetHash().ToString()); + sortedOrder.erase(sortedOrder.begin(), sortedOrder.begin()+2); // take out tx9, tx8 from the beginning + sortedOrder.insert(sortedOrder.begin()+5, tx9.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin()+6, tx8.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin()+7, tx10.GetHash().ToString()); // tx10 is just before tx6 CheckSort(pool, sortedOrder); // there should be 10 transactions in the mempool @@ -281,4 +281,157 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort(pool, snapshotOrder); } +BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) +{ + CTxMemPool pool(CFeeRate(1000)); + + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vin.resize(1); + tx1.vin[0].scriptSig = CScript() << OP_1; + tx1.vout.resize(1); + tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx1.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx1.GetHash(), CTxMemPoolEntry(tx1, 10000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx1))); + + CMutableTransaction tx2 = CMutableTransaction(); + tx2.vin.resize(1); + tx2.vin[0].scriptSig = CScript() << OP_2; + tx2.vout.resize(1); + tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; + tx2.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx2.GetHash(), CTxMemPoolEntry(tx2, 5000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx2))); + + pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing + BOOST_CHECK(pool.exists(tx1.GetHash())); + BOOST_CHECK(pool.exists(tx2.GetHash())); + + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // should remove the lower-feerate transaction + BOOST_CHECK(pool.exists(tx1.GetHash())); + BOOST_CHECK(!pool.exists(tx2.GetHash())); + + pool.addUnchecked(tx2.GetHash(), CTxMemPoolEntry(tx2, 5000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx2))); + CMutableTransaction tx3 = CMutableTransaction(); + tx3.vin.resize(1); + tx3.vin[0].prevout = COutPoint(tx2.GetHash(), 0); + tx3.vin[0].scriptSig = CScript() << OP_2; + tx3.vout.resize(1); + tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; + tx3.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx3.GetHash(), CTxMemPoolEntry(tx3, 20000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx3))); + + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) + BOOST_CHECK(!pool.exists(tx1.GetHash())); + BOOST_CHECK(pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(tx3.GetHash())); + + pool.TrimToSize(::GetSerializeSize(CTransaction(tx1), SER_NETWORK, PROTOCOL_VERSION)); // mempool is limited to tx1's size in memory usage, so nothing fits + BOOST_CHECK(!pool.exists(tx1.GetHash())); + BOOST_CHECK(!pool.exists(tx2.GetHash())); + BOOST_CHECK(!pool.exists(tx3.GetHash())); + + CFeeRate maxFeeRateRemoved(25000, ::GetSerializeSize(CTransaction(tx3), SER_NETWORK, PROTOCOL_VERSION) + ::GetSerializeSize(CTransaction(tx2), SER_NETWORK, PROTOCOL_VERSION)); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); + + CMutableTransaction tx4 = CMutableTransaction(); + tx4.vin.resize(2); + tx4.vin[0].prevout.SetNull(); + tx4.vin[0].scriptSig = CScript() << OP_4; + tx4.vin[1].prevout.SetNull(); + tx4.vin[1].scriptSig = CScript() << OP_4; + tx4.vout.resize(2); + tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; + tx4.vout[0].nValue = 10 * COIN; + tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; + tx4.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx5 = CMutableTransaction(); + tx5.vin.resize(2); + tx5.vin[0].prevout = COutPoint(tx4.GetHash(), 0); + tx5.vin[0].scriptSig = CScript() << OP_4; + tx5.vin[1].prevout.SetNull(); + tx5.vin[1].scriptSig = CScript() << OP_5; + tx5.vout.resize(2); + tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; + tx5.vout[0].nValue = 10 * COIN; + tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; + tx5.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx6 = CMutableTransaction(); + tx6.vin.resize(2); + tx6.vin[0].prevout = COutPoint(tx4.GetHash(), 1); + tx6.vin[0].scriptSig = CScript() << OP_4; + tx6.vin[1].prevout.SetNull(); + tx6.vin[1].scriptSig = CScript() << OP_6; + tx6.vout.resize(2); + tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; + tx6.vout[0].nValue = 10 * COIN; + tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; + tx6.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx7 = CMutableTransaction(); + tx7.vin.resize(2); + tx7.vin[0].prevout = COutPoint(tx5.GetHash(), 0); + tx7.vin[0].scriptSig = CScript() << OP_5; + tx7.vin[1].prevout = COutPoint(tx6.GetHash(), 0); + tx7.vin[1].scriptSig = CScript() << OP_6; + tx7.vout.resize(2); + tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + + pool.addUnchecked(tx4.GetHash(), CTxMemPoolEntry(tx4, 7000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx4))); + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx6.GetHash(), CTxMemPoolEntry(tx6, 1100LL, 0, 10.0, 1, pool.HasNoInputsOf(tx6))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + // we only require this remove, at max, 2 txn, because its not clear what we're really optimizing for aside from that + pool.TrimToSize(pool.DynamicMemoryUsage() - 1); + BOOST_CHECK(pool.exists(tx4.GetHash())); + BOOST_CHECK(pool.exists(tx6.GetHash())); + BOOST_CHECK(!pool.exists(tx7.GetHash())); + + if (!pool.exists(tx5.GetHash())) + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 + BOOST_CHECK(pool.exists(tx4.GetHash())); + BOOST_CHECK(!pool.exists(tx5.GetHash())); + BOOST_CHECK(pool.exists(tx6.GetHash())); + BOOST_CHECK(!pool.exists(tx7.GetHash())); + + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + std::vector<CTransaction> vtx; + std::list<CTransaction> conflicts; + SetMockTime(42); + SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); + // ... we should keep the same min fee until we get a block + pool.removeForBlock(vtx, 1, conflicts); + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/2); + // ... then feerate should drop 1/2 each halflife + + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2); + BOOST_CHECK_EQUAL(pool.GetMinFee(pool.DynamicMemoryUsage() * 5 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/4); + // ... with a 1/2 halflife when mempool is < 1/2 its target size + + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(pool.DynamicMemoryUsage() * 9 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/8); + // ... with a 1/4 halflife when mempool is < 1/4 its target size + + SetMockTime(42 + 7*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), 1000); + // ... but feerate should never drop below 1000 + + SetMockTime(42 + 8*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), 0); + // ... unless it has gone all the way to 0 (after getting past 1000/2) + + SetMockTime(0); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 91a3a5738e..827525783a 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -4,6 +4,7 @@ #include "chainparams.h" #include "coins.h" +#include "consensus/consensus.h" #include "consensus/validation.h" #include "main.h" #include "miner.h" @@ -229,7 +230,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = chainActive.Tip()->nHeight+1; hash = tx.GetHash(); mempool.addUnchecked(hash, CTxMemPoolEntry(tx, 11, GetTime(), 111.0, 11)); - BOOST_CHECK(!CheckFinalTx(tx)); + BOOST_CHECK(!CheckFinalTx(tx, LOCKTIME_MEDIAN_TIME_PAST)); // time locked tx2.vin.resize(1); @@ -243,7 +244,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx2.nLockTime = chainActive.Tip()->GetMedianTimePast()+1; hash = tx2.GetHash(); mempool.addUnchecked(hash, CTxMemPoolEntry(tx2, 11, GetTime(), 111.0, 11)); - BOOST_CHECK(!CheckFinalTx(tx2)); + BOOST_CHECK(!CheckFinalTx(tx2, LOCKTIME_MEDIAN_TIME_PAST)); BOOST_CHECK(pblocktemplate = CreateNewBlock(scriptPubKey)); @@ -261,7 +262,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) //BOOST_CHECK(CheckFinalTx(tx2)); BOOST_CHECK(pblocktemplate = CreateNewBlock(scriptPubKey)); - BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 3); + BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 2); delete pblocktemplate; chainActive.Tip()->nHeight--; diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index edad18644e..9b8e1c088b 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx) LOCK(cs_main); CValidationState state; - return AcceptToMemoryPool(mempool, state, tx, false, NULL, false); + return AcceptToMemoryPool(mempool, state, tx, false, NULL, true, false); } BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) diff --git a/src/txdb.cpp b/src/txdb.cpp index a441aea688..cd76c0155c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch(&db.GetObfuscateKey()); + CDBBatch batch(&db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -71,7 +71,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { @@ -98,7 +98,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr<CLevelDBIterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator()); + boost::scoped_ptr<CDBIterator> pcursor(const_cast<CDBWrapper*>(&db)->NewIterator()); pcursor->Seek(DB_COINS); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -121,7 +121,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { nTotalAmount += out.nValue; } } - stats.nSerializedSize += 32 + pcursor->GetKeySize(); + stats.nSerializedSize += 32 + pcursor->GetValueSize(); ss << VARINT(0); } else { return error("CCoinsViewDB::GetStats() : unable to read value"); @@ -141,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { - CLevelDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(&GetObfuscateKey()); for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -157,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) { - CLevelDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(&GetObfuscateKey()); for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -177,7 +177,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr<CLevelDBIterator> pcursor(NewIterator()); + boost::scoped_ptr<CDBIterator> pcursor(NewIterator()); pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); diff --git a/src/txdb.h b/src/txdb.h index bef5dc9fd1..586ab55d0d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -7,7 +7,7 @@ #define BITCOIN_TXDB_H #include "coins.h" -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include <map> #include <string> @@ -26,11 +26,11 @@ static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache in (MiB) static const int64_t nMinDbCache = 4; -/** CCoinsView backed by the LevelDB coin database (chainstate/) */ +/** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView { protected: - CLevelDBWrapper db; + CDBWrapper db; public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); @@ -42,7 +42,7 @@ public: }; /** Access to the block database (blocks/index/) */ -class CBlockTreeDB : public CLevelDBWrapper +class CBlockTreeDB : public CDBWrapper { public: CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c379a45b48..a772e7adea 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -13,6 +13,7 @@ #include "streams.h" #include "util.h" #include "utilmoneystr.h" +#include "utiltime.h" #include "version.h" using namespace std; @@ -305,15 +306,18 @@ void CTxMemPoolEntry::UpdateState(int64_t modifySize, CAmount modifyFee, int64_t } } -CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : +CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : nTransactionsUpdated(0) { + _clear(); //lock free clear + // Sanity checks off by default for performance, because otherwise // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - minerPolicyEstimator = new CBlockPolicyEstimator(_minRelayFee); + minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); + minReasonableRelayFee = _minReasonableRelayFee; } CTxMemPool::~CTxMemPool() @@ -508,6 +512,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction> if (txConflict != tx) { remove(txConflict, removed, true); + ClearPrioritisation(txConflict.GetHash()); } } } @@ -538,19 +543,29 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i } // After the txs in the new block have been removed from the mempool, update policy estimates minerPolicyEstimator->processBlock(nBlockHeight, entries, fCurrentEstimate); + lastRollingFeeUpdate = GetTime(); + blockSinceLastRollingFeeBump = true; } -void CTxMemPool::clear() +void CTxMemPool::_clear() { - LOCK(cs); mapLinks.clear(); mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; cachedInnerUsage = 0; + lastRollingFeeUpdate = GetTime(); + blockSinceLastRollingFeeBump = false; + rollingMinimumFeeRate = 0; ++nTransactionsUpdated; } +void CTxMemPool::clear() +{ + LOCK(cs); + _clear(); +} + void CTxMemPool::check(const CCoinsViewCache *pcoins) const { if (nCheckFrequency == 0) @@ -738,10 +753,10 @@ void CTxMemPool::PrioritiseTransaction(const uint256 hash, const string strHash, LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) +void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const { LOCK(cs); - std::map<uint256, std::pair<double, CAmount> >::iterator pos = mapDeltas.find(hash); + std::map<uint256, std::pair<double, CAmount> >::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; const std::pair<double, CAmount> &deltas = pos->second; @@ -795,6 +810,22 @@ void CTxMemPool::RemoveStaged(setEntries &stage) { } } +int CTxMemPool::Expire(int64_t time) { + LOCK(cs); + indexed_transaction_set::nth_index<2>::type::iterator it = mapTx.get<2>().begin(); + setEntries toremove; + while (it != mapTx.get<2>().end() && it->GetTime() < time) { + toremove.insert(mapTx.project<0>(it)); + it++; + } + setEntries stage; + BOOST_FOREACH(txiter removeit, toremove) { + CalculateDescendants(removeit, stage); + } + RemoveStaged(stage); + return stage.size(); +} + bool CTxMemPool::addUnchecked(const uint256&hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) { LOCK(cs); @@ -840,3 +871,62 @@ const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) cons assert(it != mapLinks.end()); return it->second.children; } + +CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { + LOCK(cs); + if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) + return CFeeRate(rollingMinimumFeeRate); + + int64_t time = GetTime(); + if (time > lastRollingFeeUpdate + 10) { + double halflife = ROLLING_FEE_HALFLIFE; + if (DynamicMemoryUsage() < sizelimit / 4) + halflife /= 4; + else if (DynamicMemoryUsage() < sizelimit / 2) + halflife /= 2; + + rollingMinimumFeeRate = rollingMinimumFeeRate / pow(2.0, (time - lastRollingFeeUpdate) / halflife); + lastRollingFeeUpdate = time; + + if (rollingMinimumFeeRate < minReasonableRelayFee.GetFeePerK() / 2) { + rollingMinimumFeeRate = 0; + return CFeeRate(0); + } + } + return std::max(CFeeRate(rollingMinimumFeeRate), minReasonableRelayFee); +} + +void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { + AssertLockHeld(cs); + if (rate.GetFeePerK() > rollingMinimumFeeRate) { + rollingMinimumFeeRate = rate.GetFeePerK(); + blockSinceLastRollingFeeBump = false; + } +} + +void CTxMemPool::TrimToSize(size_t sizelimit) { + LOCK(cs); + + unsigned nTxnRemoved = 0; + CFeeRate maxFeeRateRemoved(0); + while (DynamicMemoryUsage() > sizelimit) { + indexed_transaction_set::nth_index<1>::type::iterator it = mapTx.get<1>().begin(); + + // We set the new mempool min fee to the feerate of the removed set, plus the + // "minimum reasonable fee rate" (ie some value under which we consider txn + // to have 0 fee). This way, we don't allow txn to enter mempool with feerate + // equal to txn which were removed with no block in between. + CFeeRate removed(it->GetFeesWithDescendants(), it->GetSizeWithDescendants()); + removed += minReasonableRelayFee; + trackPackageRemoved(removed); + maxFeeRateRemoved = std::max(maxFeeRateRemoved, removed); + + setEntries stage; + CalculateDescendants(mapTx.project<0>(it), stage); + RemoveStaged(stage); + nTxnRemoved += stage.size(); + } + + if (maxFeeRateRemoved > CFeeRate(0)) + LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); +} diff --git a/src/txmempool.h b/src/txmempool.h index 7d83decb55..6aa5f6d77c 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -83,7 +83,7 @@ public: const CTransaction& GetTx() const { return this->tx; } double GetPriority(unsigned int currentHeight) const; - CAmount GetFee() const { return nFee; } + const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return nHeight; } @@ -160,9 +160,9 @@ public: double f2 = aSize * bFees; if (f1 == f2) { - return a.GetTime() < b.GetTime(); + return a.GetTime() >= b.GetTime(); } - return f1 > f2; + return f1 < f2; } // Calculate which feerate to use for an entry (avoiding division). @@ -211,9 +211,10 @@ public: * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * - * mapTx is a boost::multi_index that sorts the mempool on 2 criteria: + * mapTx is a boost::multi_index that sorts the mempool on 3 criteria: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] + * - time in mempool * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given @@ -284,7 +285,18 @@ private: uint64_t totalTxSize; //! sum of all mempool tx' byte sizes uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves) + CFeeRate minReasonableRelayFee; + + mutable int64_t lastRollingFeeUpdate; + mutable bool blockSinceLastRollingFeeBump; + mutable double rollingMinimumFeeRate; //! minimum fee to get into the pool, decreases exponentially + + void trackPackageRemoved(const CFeeRate& rate); + public: + + static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; // public only for testing + typedef boost::multi_index_container< CTxMemPoolEntry, boost::multi_index::indexed_by< @@ -294,6 +306,11 @@ public: boost::multi_index::ordered_non_unique< boost::multi_index::identity<CTxMemPoolEntry>, CompareTxMemPoolEntryByFee + >, + // sorted by entry time + boost::multi_index::ordered_non_unique< + boost::multi_index::identity<CTxMemPoolEntry>, + CompareTxMemPoolEntryByEntryTime > > > indexed_transaction_set; @@ -328,7 +345,12 @@ public: std::map<COutPoint, CInPoint> mapNextTx; std::map<uint256, std::pair<double, CAmount> > mapDeltas; - CTxMemPool(const CFeeRate& _minRelayFee); + /** Create a new CTxMemPool. + * minReasonableRelayFee should be a feerate which is, roughly, somewhere + * around what it "costs" to relay a transaction around the network and + * below which we would reasonably say a transaction has 0-effective-fee. + */ + CTxMemPool(const CFeeRate& _minReasonableRelayFee); ~CTxMemPool(); /** @@ -353,6 +375,7 @@ public: void removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight, std::list<CTransaction>& conflicts, bool fCurrentEstimate = true); void clear(); + void _clear(); //lock free void queryHashes(std::vector<uint256>& vtxid); void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; @@ -365,7 +388,7 @@ public: /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); - void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta); + void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: @@ -397,6 +420,20 @@ public: */ bool CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true); + /** The minimum fee to get into the mempool, which may itself not be enough + * for larger-sized transactions. + * The minReasonableRelayFee constructor arg is used to bound the time it + * takes the fee rate to go back down all the way to 0. When the feerate + * would otherwise be half of this, it is set to 0 instead. + */ + CFeeRate GetMinFee(size_t sizelimit) const; + + /** Remove transactions from the mempool until its dynamic size is <= sizelimit. */ + void TrimToSize(size_t sizelimit); + + /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ + int Expire(int64_t time); + unsigned long size() { LOCK(cs); diff --git a/src/util.cpp b/src/util.cpp index 8192a7c71c..e8514a2ef0 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -108,6 +108,7 @@ bool fDaemon = false; bool fServer = false; string strMiscWarning; bool fLogTimestamps = false; +bool fLogTimeMicros = DEFAULT_LOGTIMEMICROS; bool fLogIPs = false; volatile bool fReopenDebugLog = false; CTranslationInterface translationInterface; @@ -263,9 +264,13 @@ static std::string LogTimestampStr(const std::string &str, bool *fStartedNewLine if (!fLogTimestamps) return str; - if (*fStartedNewLine) - strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime()) + ' ' + str; - else + if (*fStartedNewLine) { + int64_t nTimeMicros = GetLogTimeMicros(); + strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros/1000000); + if (fLogTimeMicros) + strStamped += strprintf(".%06d", nTimeMicros%1000000); + strStamped += ' ' + str; + } else strStamped = str; if (!str.empty() && str[str.size()-1] == '\n') diff --git a/src/util.h b/src/util.h index 0b2dc01ac6..b2779fe782 100644 --- a/src/util.h +++ b/src/util.h @@ -28,6 +28,8 @@ #include <boost/signals2/signal.hpp> #include <boost/thread/exceptions.hpp> +static const bool DEFAULT_LOGTIMEMICROS = false; + /** Signals for translation. */ class CTranslationInterface { @@ -44,6 +46,7 @@ extern bool fPrintToDebugLog; extern bool fServer; extern std::string strMiscWarning; extern bool fLogTimestamps; +extern bool fLogTimeMicros; extern bool fLogIPs; extern volatile bool fReopenDebugLog; extern CTranslationInterface translationInterface; diff --git a/src/utiltime.cpp b/src/utiltime.cpp index d316288999..3202c47f1d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -40,6 +40,14 @@ int64_t GetTimeMicros() boost::posix_time::ptime(boost::gregorian::date(1970,1,1))).total_microseconds(); } +/** Return a time useful for the debug log */ +int64_t GetLogTimeMicros() +{ + if (nMockTime) return nMockTime*1000000; + + return GetTimeMicros(); +} + void MilliSleep(int64_t n) { diff --git a/src/utiltime.h b/src/utiltime.h index 900992f871..241b5211e9 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -12,6 +12,7 @@ int64_t GetTime(); int64_t GetTimeMillis(); int64_t GetTimeMicros(); +int64_t GetLogTimeMicros(); void SetMockTime(int64_t nMockTimeIn); void MilliSleep(int64_t n); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index bd3004061b..3f2d5a05f6 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2863,6 +2863,6 @@ int CMerkleTx::GetBlocksToMaturity() const bool CMerkleTx::AcceptToMemoryPool(bool fLimitFree, bool fRejectAbsurdFee) { CValidationState state; - return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, fRejectAbsurdFee); + return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, false, fRejectAbsurdFee); } |