diff options
50 files changed, 912 insertions, 273 deletions
diff --git a/contrib/debian/changelog b/contrib/debian/changelog index bd7ab3524c..110bfe03ef 100644 --- a/contrib/debian/changelog +++ b/contrib/debian/changelog @@ -1,3 +1,33 @@ +bitcoin (0.11.0-precise1) precise; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 14 Jul 2015 14:39:00 -1000 + +bitcoin (0.10.2-precise1) precise; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Mon, 29 Jun 2015 17:33:00 -1000 + +bitcoin (0.10.1-precise3) precise; urgency=medium + + * Fix build dep (include python). + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 5 May 2015 09:28:00 -1000 + +bitcoin (0.10.1-precise2) precise; urgency=medium + + * Fix miniupnpc dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 5 May 2015 00:33:00 -1000 + +bitcoin (0.10.1-precise1) precise; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 5 May 2015 00:07:00 -1000 + bitcoin (0.10.0-precise1) precise; urgency=medium * New upstream releases. diff --git a/contrib/debian/control b/contrib/debian/control index 01625b8438..2fd68583ce 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -20,11 +20,12 @@ Build-Depends: debhelper, qt4-qmake, libqt4-dev, libqrencode-dev, - libprotobuf-dev, protobuf-compiler + libprotobuf-dev, protobuf-compiler, + python Standards-Version: 3.9.2 -Homepage: http://www.bitcoin.org/ +Homepage: https://www.bitcoin.org/ Vcs-Git: git://github.com/bitcoin/bitcoin.git -Vcs-Browser: http://github.com/bitcoin/bitcoin +Vcs-Browser: https://github.com/bitcoin/bitcoin Package: bitcoind Architecture: any diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py new file mode 100755 index 0000000000..e96eaa9c38 --- /dev/null +++ b/contrib/devtools/security-check.py @@ -0,0 +1,181 @@ +#!/usr/bin/python2 +''' +Perform basic ELF security checks on a series of executables. +Exit status will be 0 if succesful, and the program will be silent. +Otherwise the exit status will be 1 and it will log which executables failed which checks. +Needs `readelf` (for ELF) and `objdump` (for PE). +''' +from __future__ import division,print_function +import subprocess +import sys +import os + +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') + +def check_ELF_PIE(executable): + ''' + Check for position independent executable (PIE), allowing for address space randomization. + ''' + p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + ok = False + for line in stdout.split('\n'): + line = line.split() + if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': + ok = True + return ok + +def get_ELF_program_headers(executable): + '''Return type and flags for ELF program headers''' + p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + in_headers = False + count = 0 + headers = [] + for line in stdout.split('\n'): + if line.startswith('Program Headers:'): + in_headers = True + if line == '': + in_headers = False + if in_headers: + if count == 1: # header line + ofs_typ = line.find('Type') + ofs_offset = line.find('Offset') + ofs_flags = line.find('Flg') + ofs_align = line.find('Align') + if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: + raise ValueError('Cannot parse elfread -lW output') + elif count > 1: + typ = line[ofs_typ:ofs_offset].rstrip() + flags = line[ofs_flags:ofs_align].rstrip() + headers.append((typ, flags)) + count += 1 + return headers + +def check_ELF_NX(executable): + ''' + Check that no sections are writable and executable (including the stack) + ''' + have_wx = False + have_gnu_stack = False + for (typ, flags) in get_ELF_program_headers(executable): + if typ == 'GNU_STACK': + have_gnu_stack = True + if 'W' in flags and 'E' in flags: # section is both writable and executable + have_wx = True + return have_gnu_stack and not have_wx + +def check_ELF_RELRO(executable): + ''' + Check for read-only relocations. + GNU_RELRO program header must exist + Dynamic section must have BIND_NOW flag + ''' + have_gnu_relro = False + for (typ, flags) in get_ELF_program_headers(executable): + # Note: not checking flags == 'R': here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + # However, the dynamic linker need to write to this area so these are RW. + # Glibc itself takes care of mprotecting this area R after relocations are finished. + # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 + if typ == 'GNU_RELRO': + have_gnu_relro = True + + have_bindnow = False + p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>1 and tokens[1] == '(BIND_NOW)': + have_bindnow = True + return have_gnu_relro and have_bindnow + +def check_ELF_Canary(executable): + ''' + Check for use of stack canary + ''' + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + ok = False + for line in stdout.split('\n'): + if '__stack_chk_fail' in line: + ok = True + return ok + +def get_PE_dll_characteristics(executable): + ''' + Get PE DllCharacteristics bits + ''' + p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': + return int(tokens[1],16) + return 0 + + +def check_PE_PIE(executable): + '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + return bool(get_PE_dll_characteristics(executable) & 0x40) + +def check_PE_NX(executable): + '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' + return bool(get_PE_dll_characteristics(executable) & 0x100) + +CHECKS = { +'ELF': [ + ('PIE', check_ELF_PIE), + ('NX', check_ELF_NX), + ('RELRO', check_ELF_RELRO), + ('Canary', check_ELF_Canary) +], +'PE': [ + ('PIE', check_PE_PIE), + ('NX', check_PE_NX) +] +} + +def identify_executable(executable): + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + return None + +if __name__ == '__main__': + retval = 0 + for filename in sys.argv[1:]: + try: + etype = identify_executable(filename) + if etype is None: + print('%s: unknown format' % filename) + retval = 1 + continue + + failed = [] + for (name, func) in CHECKS[etype]: + if not func(filename): + failed.append(name) + if failed: + print('%s: failed %s' % (filename, ' '.join(failed))) + retval = 1 + except IOError: + print('%s: cannot open' % filename) + retval = 1 + exit(retval) + diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index fad891f800..34f1ed2d17 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ from __future__ import division, print_function import subprocess import re import sys +import os # Debian 6.0.9 (Squeeze) has: # @@ -45,8 +46,10 @@ MAX_VERSIONS = { IGNORE_EXPORTS = { '_edata', '_end', '_init', '__bss_start', '_fini' } -READELF_CMD = '/usr/bin/readelf' -CPPFILT_CMD = '/usr/bin/c++filt' +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = {'librt.so.1','libpthread.so.0','libanl.so.1','libm.so.6','libgcc_s.so.1','libc.so.6','ld-linux-x86-64.so.2'} class CPPFilt(object): ''' @@ -98,6 +101,22 @@ def check_version(max_versions, version): return False return ver <= max_versions[lib] +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 @@ -113,6 +132,11 @@ if __name__ == '__main__': continue print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py new file mode 100755 index 0000000000..fed7626aab --- /dev/null +++ b/contrib/devtools/test-security-check.py @@ -0,0 +1,60 @@ +#!/usr/bin/python2 +''' +Test script for security-check.py +''' +from __future__ import division,print_function +import subprocess +import sys +import unittest + +def write_testcode(filename): + with open(filename, 'w') as f: + f.write(''' + #include <stdio.h> + int main() + { + printf("the quick brown fox jumps over the lazy god\\n"); + return 0; + } + ''') + +def call_security_check(cc, source, executable, options): + subprocess.check_call([cc,source,'-o',executable] + options) + p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + return (p.returncode, stdout.rstrip()) + +class TestSecurityChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = 'gcc' + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + (0, '')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = 'i686-w64-mingw32-gcc' + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, []), + (1, executable+': failed PIE NX')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), + (0, '')) + +if __name__ == '__main__': + unittest.main() + diff --git a/depends/config.guess b/depends/config.guess index f357ec6c8f..3c022c5bb8 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -2,7 +2,7 @@ # Attempt to guess a canonical system name. # Copyright 1992-2015 Free Software Foundation, Inc. -timestamp='2015-03-04' +timestamp='2015-09-14' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -221,7 +221,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: @@ -249,6 +249,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-unknown-sortix + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -962,6 +965,9 @@ EOF ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -1038,7 +1044,7 @@ EOF echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} @@ -1117,7 +1123,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configure will decide that + # prints for the "djgpp" host, or else GDB configury will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; diff --git a/depends/config.sub b/depends/config.sub index 8f1229c6f7..1acc966a33 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -2,7 +2,7 @@ # Configuration validation subroutine script. # Copyright 1992-2015 Free Software Foundation, Inc. -timestamp='2015-03-08' +timestamp='2015-08-20' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -255,6 +255,7 @@ case $basic_machine in | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ + | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ @@ -305,7 +306,7 @@ case $basic_machine in | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -376,6 +377,7 @@ case $basic_machine in | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ @@ -428,12 +430,13 @@ case $basic_machine in | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ + | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ @@ -1376,7 +1379,7 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* | -cloudabi* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk index 22b1017ffc..687aae6682 100644 --- a/depends/packages/openssl.mk +++ b/depends/packages/openssl.mk @@ -10,7 +10,7 @@ $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/o $(package)_config_opts+=no-krb5 no-camellia no-capieng no-cast no-cms no-dtls1 no-gost no-gmp no-heartbeats no-idea no-jpake no-md2 $(package)_config_opts+=no-mdc2 no-rc5 no-rdrand no-rfc3779 no-rsax no-sctp no-seed no-sha0 no-static_engine no-whirlpool no-rc2 no-rc4 no-ssl2 no-ssl3 $(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) -$(package)_config_opts_linux=-fPIC +$(package)_config_opts_linux=-fPIC -Wa,--noexecstack $(package)_config_opts_x86_64_linux=linux-x86_64 $(package)_config_opts_i686_linux=linux-generic32 $(package)_config_opts_arm_linux=linux-generic32 diff --git a/doc/README.md b/doc/README.md index 3e3f4a294f..08fd724355 100644 --- a/doc/README.md +++ b/doc/README.md @@ -7,7 +7,7 @@ Setup Running --------------------- -The following are some helpful notes on how to run Bitcoin on your native platform. +The following are some helpful notes on how to run Bitcoin on your native platform. ### Unix @@ -26,7 +26,7 @@ Unpack the files into a directory and run: Unpack the files into a directory, and then run bitcoin-qt.exe. -### OSX +### OS X Drag Bitcoin-Qt to your applications folder, and then run Bitcoin-Qt. @@ -41,7 +41,7 @@ Building --------------------- The following are developer notes on how to build Bitcoin on your native platform. They are not complete guides, but include notes on the necessary libraries, compile flags, etc. -- [OSX Build Notes](build-osx.md) +- [OS X Build Notes](build-osx.md) - [Unix Build Notes](build-unix.md) - [Gitian Building Guide](gitian-building.md) diff --git a/doc/README_osx.txt b/doc/README_osx.txt index a572c7a241..f589bfc676 100644 --- a/doc/README_osx.txt +++ b/doc/README_osx.txt @@ -1,12 +1,12 @@ -Deterministic OSX Dmg Notes. +Deterministic OS X Dmg Notes. -Working OSX DMGs are created in Linux by combining a recent clang, +Working OS X DMGs are created in Linux by combining a recent clang, the Apple's binutils (ld, ar, etc), and DMG authoring tools. Apple uses clang extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of -F, -target, -mmacosx-version-min, and --sysroot, which are all necessary -when building for OSX. A pre-compiled version of 3.2 is used because it was not +when building for OS X. A pre-compiled version of 3.2 is used because it was not available in the Precise repositories at the time this work was started. In the future, it can be switched to use system packages instead. @@ -29,18 +29,18 @@ originally done in toolchain4. To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. -To obtain it, register for a developer account, then download the XCode 6.1.1 dmg: +To obtain it, register for a developer account, then download the Xcode 6.1.1 dmg: https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg This file is several gigabytes in size, but only a single directory inside is needed: Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.9.sdk Unfortunately, the usual linux tools (7zip, hpmount, loopback mount) are incapable of opening this file. -To create a tarball suitable for gitian input, mount the dmg in OSX, then create it with: +To create a tarball suitable for Gitian input, mount the dmg in OS X, then create it with: $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk -The gitian descriptors build 2 sets of files: Linux tools, then Apple binaries +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are created using these tools. The build process has been designed to avoid including the SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely redistributed. @@ -64,20 +64,20 @@ Ideally, the creation could be fixed and genisoimage would no longer be necessar Background images and other features can be added to DMG files by inserting a .DS_Store before creation. The easiest way to create this file is to build a -DMG without one, move it to a device running OSX, customize the layout, then +DMG without one, move it to a device running OS X, customize the layout, then grab the .DS_Store file for later use. That is the approach taken here. -As of OSX Mavericks (10.9), using an Apple-blessed key to sign binaries is a +As of OS X Mavericks (10.9), using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use gitian to create an unsigned release. This outputs an unsigned +- Builders use Gitian to create an unsigned release. This outputs an unsigned dmg which users may choose to bless and run. It also outputs an unsigned app structure in the form of a tarball, which also contains all of the tools that have been previously (deterministically) built in order to create a final dmg. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. -- Builders feed the unsigned app + detached signature back into gitian. It +- Builders feed the unsigned app + detached signature back into Gitian. It uses the pre-built tools to recombine the pieces into a deterministic dmg. diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index a26b52465e..d923301467 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -38,7 +38,7 @@ Do not use `pkg_add boost`! The boost version installed thus is compiled using t test_bitcoin:/usr/lib/libstdc++.so.57.0: /usr/local/lib/libestdc++.so.17.0 : WARNING: symbol(_ZN11__gnu_debug17_S_debug_me ssagesE) size mismatch, relink your program ... - Segmentation fault (core dumped) + Segmentation fault (core dumped) This makes it necessary to build boost, or at least the parts used by Bitcoin Core, manually: @@ -57,7 +57,7 @@ tar -xjf boost_1_59_0.tar.bz2 # Boost 1.59 needs two small patches for OpenBSD cd boost_1_59_0 # Also here: https://gist.githubusercontent.com/laanwj/bf359281dc319b8ff2e1/raw/92250de8404b97bb99d72ab898f4a8cb35ae1ea3/patch-boost_test_impl_execution_monitor_ipp.patch -patch -p0 < /usr/ports/devel/boost/patches/patch-boost_test_impl_execution_monitor_ipp +patch -p0 < /usr/ports/devel/boost/patches/patch-boost_test_impl_execution_monitor_ipp # https://github.com/boostorg/filesystem/commit/90517e459681790a091566dce27ca3acabf9a70c sed 's/__OPEN_BSD__/__OpenBSD__/g' < libs/filesystem/src/path.cpp > libs/filesystem/src/path.cpp.tmp mv libs/filesystem/src/path.cpp.tmp libs/filesystem/src/path.cpp @@ -92,7 +92,7 @@ tar -xzf db-4.8.30.NC.tar.gz # Build the library and install to specified prefix cd db-4.8.30.NC/build_unix/ # Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX CC=egcc CXX=eg++ CPP=ecpp +../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX CC=egcc CXX=eg++ CPP=ecpp make install ``` @@ -160,4 +160,3 @@ version installed by OpenBSD 5.7: - https://llvm.org/bugs/show_bug.cgi?id=9758 There is no known workaround for this. - diff --git a/doc/build-osx.md b/doc/build-osx.md index 201fe9522b..69c401b751 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,6 +1,6 @@ Mac OS X Build Instructions and Notes ==================================== -This guide will show you how to build bitcoind (headless client) for OSX. +This guide will show you how to build bitcoind (headless client) for OS X. Notes ----- @@ -13,8 +13,8 @@ built-in one is located in `/Applications/Utilities`. Preparation ----------- -You need to install XCode with all the options checked so that the compiler -and everything is available in /usr not just /Developer. XCode should be +You need to install Xcode with all the options checked so that the compiler +and everything is available in /usr not just /Developer. Xcode should be available on your OS X installation media, but if not, you can get the current version from https://developer.apple.com/xcode/. If you install Xcode 4.3 or later, you'll need to install its command line tools. This can @@ -38,7 +38,7 @@ NOTE: Building with Qt4 is still supported, however, could result in a broken UI ### Building `bitcoind` -1. Clone the github tree to get the source code and go into the directory. +1. Clone the GitHub tree to get the source code and go into the directory. git clone https://github.com/bitcoin/bitcoin.git cd bitcoin @@ -62,7 +62,7 @@ Use Qt Creator as IDE You can use Qt Creator as IDE, for debugging and for manipulating forms, etc. Download Qt Creator from http://www.qt.io/download/. Download the "community edition" and only install Qt Creator (uncheck the rest during the installation process). -1. Make sure you installed everything through homebrew mentioned above +1. Make sure you installed everything through Homebrew mentioned above 2. Do a proper ./configure --with-gui=qt5 --enable-debug 3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project 4. Enter "bitcoin-qt" as project name, enter src/qt as location diff --git a/doc/developer-notes.md b/doc/developer-notes.md index a649b3af4d..4189d22187 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -57,7 +57,7 @@ As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this cas To describe a class use the same construct above the class definition: ```c++ -/** +/** * Alerts are for notifying old versions if they become too obsolete and * need to upgrade. The message is displayed in the status bar. * @see GetWarnings() diff --git a/doc/dnsseed-policy.md b/doc/dnsseed-policy.md index 814ae3876a..55a5c28258 100644 --- a/doc/dnsseed-policy.md +++ b/doc/dnsseed-policy.md @@ -7,7 +7,7 @@ As such, DNS seeds must be run by entities which have some minimum level of trust within the Bitcoin community. Other implementations of Bitcoin software may also use the same -seeds and may be more exposed. In light of this exposure, this +seeds and may be more exposed. In light of this exposure, this document establishes some basic expectations for operating dnsseeds. 0. A DNS seed operating organization or person is expected to follow good diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 265fbd586d..00fdce82e8 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -1,7 +1,7 @@ Gitian building ================ -*Setup instructions for a gitian build of Bitcoin using a Debian VM or physical system.* +*Setup instructions for a Gitian build of Bitcoin using a Debian VM or physical system.* Gitian is the deterministic build process that is used to build the Bitcoin Core executables. It provides a way to be reasonably sure that the @@ -13,7 +13,7 @@ Multiple developers build the source code by following a specific descriptor These results are compared and only if they match, the build is accepted and uploaded to bitcoin.org. -More independent gitian builders are needed, which is why this guide exists. +More independent Gitian builders are needed, which is why this guide exists. It is preferred you follow these steps yourself instead of using someone else's VM image to avoid 'contaminating' the build. @@ -22,9 +22,9 @@ Table of Contents - [Create a new VirtualBox VM](#create-a-new-virtualbox-vm) - [Connecting to the VM](#connecting-to-the-vm) -- [Setting up Debian for gitian building](#setting-up-debian-for-gitian-building) -- [Installing gitian](#installing-gitian) -- [Setting up the gitian image](#setting-up-the-gitian-image) +- [Setting up Debian for Gitian building](#setting-up-debian-for-gitian-building) +- [Installing Gitian](#installing-gitian) +- [Setting up the Gitian image](#setting-up-the-gitian-image) - [Getting and building the inputs](#getting-and-building-the-inputs) - [Building Bitcoin](#building-bitcoin) - [Building an alternative repository](#building-an-alternative-repository) @@ -43,7 +43,7 @@ Any kind of virtualization can be used, for example: - [KVM](http://www.linux-kvm.org/page/Main_Page) - [LXC](https://linuxcontainers.org/), see also [Gitian host docker container](https://github.com/gdm85/tenku/tree/master/docker/gitian-bitcoin-host/README.md). -You can also install gitian on actual hardware instead of using virtualization. +You can also install Gitian on actual hardware instead of using virtualization. Create a new VirtualBox VM --------------------------- @@ -60,18 +60,18 @@ In the VirtualBox GUI click "Create" and choose the following parameters in the  - Hard Disk: Create a virtual hard disk now - +  -- Hard Disk file type: Use the default, VDI (VirtualBox Disk Image) +- Hard Disk file type: Use the default, VDI (VirtualBox Disk Image)  - -- Storage on physical hard disk: Dynamically Allocated - + +- Storage on physical hard disk: Dynamically Allocated +  -- File location and size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side +- File location and size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side - Click `Create` Get the [Debian 8.x net installer](http://cdimage.debian.org/debian-cd/8.2.0/amd64/iso-cd/debian-8.2.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). @@ -81,7 +81,7 @@ Unixy OSes by entering the following in a terminal: echo "d393d17ac6b3113c81186e545c416a00f28ed6e05774284bb5e8f0df39fcbcb9 debian-8.2.0-amd64-netinst.iso" | sha256sum -c # (must return OK) -After creating the VM, we need to configure it. +After creating the VM, we need to configure it. - Click the `Settings` button, then go to the `Network` tab. Adapter 1 should be attached to `NAT`. @@ -115,8 +115,8 @@ This section will explain how to install Debian on the newly created VM.  -**Note**: Navigating in the Debian installer: -To keep a setting at the default and proceed, just press `Enter`. +**Note**: Navigating in the Debian installer: +To keep a setting at the default and proceed, just press `Enter`. To select a different button, press `Tab`. - Choose locale and keyboard settings (doesn't matter, you can just go with the defaults or select your own information) @@ -126,23 +126,23 @@ To select a different button, press `Tab`.  - The VM will detect network settings using DHCP, this should all proceed automatically -- Configure the network: +- Configure the network: - Hostname `debian`. - Leave domain name empty.  -- Choose a root password and enter it twice (remember it for later) +- Choose a root password and enter it twice (remember it for later)  -- Name the new user `debian` (the full name doesn't matter, you can leave it empty) +- Name the new user `debian` (the full name doesn't matter, you can leave it empty) - Set the account username as `debian`   -- Choose a user password and enter it twice (remember it for later) +- Choose a user password and enter it twice (remember it for later)  @@ -152,11 +152,11 @@ To select a different button, press `Tab`.  - Disk setup - - Partitioning method: Guided - Use the entire disk - + - Partitioning method: Guided - Use the entire disk +  - - Select disk to partition: SCSI1 (0,0,0) + - Select disk to partition: SCSI1 (0,0,0)  @@ -166,7 +166,7 @@ To select a different button, press `Tab`.  - The base system will be installed, this will take a minute or so -- Choose a mirror (any will do) +- Choose a mirror (any will do)  @@ -201,7 +201,7 @@ After Installation The next step in the guide involves logging in as root via SSH. SSH login for root users is disabled by default, so we'll enable that now. -Login to the VM using username `root` and the root password you choose earlier. +Login to the VM using username `root` and the root password you chose earlier. You'll be presented with a screen similar to this.  @@ -243,7 +243,7 @@ For example, to connect as `root` from a Linux command prompt use Replace `root` with `debian` to log in as user. -Setting up Debian for gitian building +Setting up Debian for Gitian building -------------------------------------- In this section we will be setting up the Debian installation for Gitian building. @@ -260,7 +260,7 @@ Then set up LXC and the rest with the following, which is a complex jumble of se ```bash # the version of lxc-start in Debian 7.4 needs to run as root, so make sure -# that the build script can exectute it without providing a password +# that the build script can execute it without providing a password echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc # add cgroup for LXC echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab @@ -280,7 +280,7 @@ reboot At the end the VM is rebooted to make sure that the changes take effect. The steps in this section only need to be performed once. -Installing gitian +Installing Gitian ------------------ Re-login as the user `debian` that was created during installation. @@ -300,14 +300,14 @@ cd .. **Note**: When sudo asks for a password, enter the password for the user *debian* not for *root*. -Clone the git repositories for bitcoin and gitian. +Clone the git repositories for bitcoin and Gitian. ```bash git clone https://github.com/devrandom/gitian-builder.git git clone https://github.com/bitcoin/bitcoin ``` -Setting up the gitian image +Setting up the Gitian image ------------------------- Gitian needs a virtual image of the operating system to build in. @@ -333,14 +333,14 @@ Getting and building the inputs Follow the instructions in [doc/release-process.md](release-process.md#fetch-and-build-inputs-first-time-or-when-dependency-versions-change) in the bitcoin repository under 'Fetch and build inputs' to install sources which require manual intervention. Also optionally follow the next step: 'Seed the Gitian sources cache -and offline git repositories' which will fetch the remaining files required for building +and offline git repositories' which will fetch the remaining files required for building offline. Building Bitcoin ---------------- -To build Bitcoin (for Linux, OSX and Windows) just follow the steps under 'perform -gitian builds' in [doc/release-process.md](release-process.md#perform-gitian-builds) in the bitcoin repository. +To build Bitcoin (for Linux, OS X and Windows) just follow the steps under 'perform +Gitian builds' in [doc/release-process.md](release-process.md#perform-gitian-builds) in the bitcoin repository. This may take some time as it will build all the dependencies needed for each descriptor. These dependencies will be cached after a successful build to avoid rebuilding them when possible. @@ -380,7 +380,7 @@ Building an alternative repository ----------------------------------- If you want to do a test build of a pull on GitHub it can be useful to point -the gitian builder at an alternative repository, using the same descriptors +the Gitian builder at an alternative repository, using the same descriptors and inputs. For example: @@ -395,9 +395,9 @@ COMMIT=2014_03_windows_unicode_path Building fully offline ----------------------- -For building fully offline including attaching signatures to unsigned builds, the detached-sigs repository +For building fully offline including attaching signatures to unsigned builds, the detached-sigs repository and the bitcoin git repository with the desired tag must both be available locally, and then gbuild must be -told where to find them. It also requires an apt-cacher-ng which is fully-populated but set to offline mode, or +told where to find them. It also requires an apt-cacher-ng which is fully-populated but set to offline mode, or manually disabling gitian-builder's use of apt-get to update the VM build environment. To configure apt-cacher-ng as an offline cacher, you will need to first populate its cache with the relevant @@ -417,7 +417,7 @@ LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root \ -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install \ $( sed -ne '/^packages:/,/[^-] .*/ {/^- .*/{s/"//g;s/- //;p}}' ../bitcoin/contrib/gitian-descriptors/*|sort|uniq ) LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root apt-get -q -y purge grub -LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade +LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade ``` And then set offline mode for apt-cacher-ng: @@ -431,7 +431,7 @@ Offlinemode: 1 service apt-cacher-ng restart ``` -Then when building, override the remote URLs that gbuild would otherwise pull from the gitian descriptors:: +Then when building, override the remote URLs that gbuild would otherwise pull from the Gitian descriptors:: ```bash cd /some/root/path/ @@ -461,7 +461,7 @@ in `gitian.sigs` to your signing machine and do ``` This will create the `.sig` files that can be committed together with the `.assert` files to assert your -gitian build. +Gitian build. Uploading signatures --------------------- diff --git a/doc/init.md b/doc/init.md index ed9ce72154..d24c2d1dbf 100644 --- a/doc/init.md +++ b/doc/init.md @@ -29,20 +29,20 @@ file, however it is recommended that a strong and secure password be used as this password is security critical to securing the wallet should the wallet be enabled. -If bitcoind is run with the "-server" flag (set by default), and no rpcpassword is set, -it will use a special cookie file for authentication. The cookie is generated with random +If bitcoind is run with the "-server" flag (set by default), and no rpcpassword is set, +it will use a special cookie file for authentication. The cookie is generated with random content when the daemon starts, and deleted when it exits. Read access to this file -controls who can access it through RPC. +controls who can access it through RPC. -By default the cookie is stored in the data directory, but it's location can be overridden +By default the cookie is stored in the data directory, but it's location can be overridden with the option '-rpccookiefile'. This allows for running bitcoind without having to do any manual configuration. -`conf`, `pid`, and `wallet` accept relative paths which are interpreted as +`conf`, `pid`, and `wallet` accept relative paths which are interpreted as relative to the data directory. `wallet` *only* supports relative paths. -For an example configuration file that describes the configuration settings, +For an example configuration file that describes the configuration settings, see `contrib/debian/examples/bitcoin.conf`. 3. Paths @@ -93,8 +93,8 @@ use old versions of Upstart and do not supply the start-stop-daemon utility. Copy bitcoind.init to /etc/init.d/bitcoind. Test by running `service bitcoind start`. -Using this script, you can adjust the path and flags to the bitcoind program by -setting the BITCOIND and FLAGS environment variables in the file +Using this script, you can adjust the path and flags to the bitcoind program by +setting the BITCOIND and FLAGS environment variables in the file /etc/sysconfig/bitcoind. You can also use the DAEMONOPTS environment variable here. 5. Auto-respawn @@ -102,4 +102,3 @@ setting the BITCOIND and FLAGS environment variables in the file Auto respawning is currently only configured for Upstart and systemd. Reasonable defaults have been chosen but YMMV. - diff --git a/doc/release-notes.md b/doc/release-notes.md index 78ab3516f6..e9b2d75a76 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -181,7 +181,7 @@ configured specifically to process scriptPubKey and not scriptSig scripts. - Removed bitrpc.py from contrib -Addition of ZMQ-based Notifcations +Addition of ZMQ-based Notifications ================================== Bitcoind can now (optionally) asynchronously notify clients through a diff --git a/doc/release-process.md b/doc/release-process.md index a562c98dbe..9a2362cb85 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -6,7 +6,7 @@ Release Process * * * -###First time / New builders +###First time / New builders Check out the source code in the following directory hierarchy. cd /path/to/your/toplevel/build @@ -34,17 +34,17 @@ Check out the source code in the following directory hierarchy. * * * -###Setup and perform gitian builds +###Setup and perform Gitian builds + + Setup Gitian descriptors: - Setup gitian descriptors: - pushd ./bitcoin - export SIGNER=(your gitian key, ie bluematt, sipa, etc) + export SIGNER=(your Gitian key, ie bluematt, sipa, etc) export VERSION=(new version, e.g. 0.8.0) git checkout v${VERSION} popd - Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other gitian signatures. + Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other Gitian signatures. pushd ./gitian.sigs git pull @@ -56,35 +56,35 @@ Check out the source code in the following directory hierarchy. git pull ###Fetch and create inputs: (first time, or when dependency versions change) - + mkdir -p inputs wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch wget -P inputs http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz - Register and download the Apple SDK: see [OSX readme](README_osx.txt) for details. - + Register and download the Apple SDK: see [OS X readme](README_osx.txt) for details. + https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg - + Using a Mac, create a tarball for the 10.9 SDK and copy it to the inputs directory: - + tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk ###Optional: Seed the Gitian sources cache and offline git repositories -By default, gitian will fetch source files as needed. To cache them ahead of time: +By default, Gitian will fetch source files as needed. To cache them ahead of time: make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common Only missing files will be fetched, so this is safe to re-run for each build. -NOTE: Offline builds must use the --url flag to ensure gitian fetches only from local URLs. For example: +NOTE: Offline builds must use the --url flag to ensure Gitian fetches only from local URLs. For example: ``` -./bin/bguild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} +./bin/gbuild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} ``` The gbuild invocations below <b>DO NOT DO THIS</b> by default. ###Build (and optionally verify) Bitcoin Core for Linux, Windows, and OS X: - + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml @@ -108,8 +108,8 @@ The gbuild invocations below <b>DO NOT DO THIS</b> by default. 1. source tarball (bitcoin-${VERSION}.tar.gz) 2. linux 32-bit and 64-bit dist tarballs (bitcoin-${VERSION}-linux[32|64].tar.gz) 3. windows 32-bit and 64-bit unsigned installers and dist zips (bitcoin-${VERSION}-win[32|64]-setup-unsigned.exe, bitcoin-${VERSION}-win[32|64].zip) - 4. OSX unsigned installer and dist tarball (bitcoin-${VERSION}-osx-unsigned.dmg, bitcoin-${VERSION}-osx64.tar.gz) - 5. Gitian signatures (in gitian.sigs/${VERSION}-<linux|{win,osx}-unsigned>/(your gitian key)/ + 4. OS X unsigned installer and dist tarball (bitcoin-${VERSION}-osx-unsigned.dmg, bitcoin-${VERSION}-osx64.tar.gz) + 5. Gitian signatures (in gitian.sigs/${VERSION}-<linux|{win,osx}-unsigned>/(your Gitian key)/ ###Next steps: @@ -123,12 +123,12 @@ Commit your signature to gitian.sigs: git push # Assuming you can push to the gitian.sigs tree popd - Wait for Windows/OSX detached signatures: + Wait for Windows/OS X detached signatures: - Once the Windows/OSX builds each have 3 matching signatures, they will be signed with their respective release keys. + Once the Windows/OS X builds each have 3 matching signatures, they will be signed with their respective release keys. Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. - Create (and optionally verify) the signed OSX binary: + Create (and optionally verify) the signed OS X binary: pushd ./gitian-builder ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -147,7 +147,7 @@ Commit your signature to gitian.sigs: mv build/out/bitcoin-*win32-setup.exe ../bitcoin-${VERSION}-win32-setup.exe popd -Commit your signature for the signed OSX/Windows binaries: +Commit your signature for the signed OS X/Windows binaries: pushd gitian.sigs git add ${VERSION}-osx-signed/${SIGNER} diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md index f1448f7258..f4ff53d6e9 100644 --- a/doc/shared-libraries.md +++ b/doc/shared-libraries.md @@ -41,3 +41,4 @@ The interface is defined in the C header `bitcoinconsensus.h` located in `src/s - [NBitcoin](https://github.com/NicolasDorier/NBitcoin/blob/master/NBitcoin/Script.cs#L814) (.NET Bindings) - [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus) (Node.js Bindings) - [java-libbitcoinconsensus](https://github.com/dexX7/java-libbitcoinconsensus) (Java Bindings) +- [bitcoinconsensus-php](https://github.com/Bit-Wasp/bitcoinconsensus-php) (PHP Bindings) diff --git a/doc/tor.md b/doc/tor.md index f8b94d19d1..594897f896 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -15,15 +15,15 @@ outgoing connections be anonymized, but more is possible. -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy server will be used to try to reach .onion addresses as well. - + -onion=ip:port Set the proxy server to use for tor hidden services. You do not need to set this if it's the same as -proxy. You can use -noonion to explicitly disable access to hidden service. - + -listen When using -proxy, listening is disabled by default. If you want to run a hidden service (see next section), you'll need to enable it explicitly. - + -connect=X When behind a Tor proxy, you can specify .onion addresses instead -addnode=X of IP addresses or hostnames in these parameters. It requires -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with @@ -55,10 +55,10 @@ your bitcoind's P2P listen port (8333 by default). preference for your node to advertize itself with, for connections coming from unroutable addresses (such as 127.0.0.1, where the Tor proxy typically runs). - + -listen You'll need to enable listening for incoming connections, as this is off by default behind a proxy. - + -discover When -externalip is specified, no attempt is made to discover local IPv4 or IPv6 addresses. If you want to run a dual stack, reachable from both Tor and IPv4 (or IPv6), you'll need to either pass your @@ -87,4 +87,3 @@ If you only want to use Tor to reach onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: ./bitcoin -onion=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -discover - diff --git a/doc/translation_process.md b/doc/translation_process.md index b06975e1d8..6389c5aced 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -1,7 +1,7 @@ Translations ============ -The Bitcoin-Core project has been designed to support multiple localisations. This makes adding new phrases, and completely new languages easily achievable. For managing all application translations, Bitcoin-Core makes use of the Transifex online translation management tool. +The Bitcoin-Core project has been designed to support multiple localisations. This makes adding new phrases, and completely new languages easily achievable. For managing all application translations, Bitcoin-Core makes use of the Transifex online translation management tool. ### Helping to translate (using Transifex) Transifex is setup to monitor the Github repo for updates, and when code containing new translations is found, Transifex will process any changes. It may take several hours after a pull-request has been merged, to appear in the Transifex web interface. diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index cf72a55b20..936a6112c6 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -1,7 +1,7 @@ Translation Strings Policy =========================== -This document provides guidelines for internationalization of the Bitcoin Core software. +This document provides guidelines for internationalization of the Bitcoin Core software. How to translate? ------------------ @@ -107,4 +107,3 @@ The second example reduces the number of pluralized words that translators have During a string freeze (often before a major release), no translation strings are to be added, modified or removed. This can be checked by executing `make translate` in the `src` directory, then verifying that `bitcoin_en.ts` remains unchanged. - diff --git a/doc/zmq.md b/doc/zmq.md index 53e557e3db..902d1124c7 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -2,7 +2,7 @@ [ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP connections, inter-process communication, and shared-memory, -providing various message-oriented semantics such as publish/subcribe, +providing various message-oriented semantics such as publish/subscribe, request/reply, and push/pull. The Bitcoin Core daemon can be configured to act as a trusted "border diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index db97767272..8a431d718d 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -81,12 +81,11 @@ testScriptsExt = [ 'invalidateblock.py', 'keypool.py', 'receivedby.py', - 'rpcbind_test.py', -# 'script_test.py', +# 'rpcbind_test.py', #temporary, bug in libevent, see #6655 +# 'script_test.py', #used for manual comparison of 2 binaries 'smartfees.py', 'maxblocksinflight.py', 'invalidblockrequest.py', -# 'forknotify.py', 'p2p-acceptblock.py', 'mempool_packages.py', ] @@ -102,7 +101,7 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts or testScripts[i] in opts or re.sub(".py$", "", testScripts[i]) in opts ): print "Running testscript " + testScripts[i] + "..." - subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) + subprocess.check_call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) #exit if help is called so we print just one set of instructions p = re.compile(" -h| --help") if p.match(passOn): @@ -113,6 +112,6 @@ if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): if ('-extended' in opts or testScriptsExt[i] in opts or re.sub(".py$", "", testScriptsExt[i]) in opts): print "Running 2nd level testscript " + testScriptsExt[i] + "..." - subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) + subprocess.check_call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) else: print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" diff --git a/qa/rpc-tests/fundrawtransaction.py b/qa/rpc-tests/fundrawtransaction.py index fc29789218..93d13faa06 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/qa/rpc-tests/fundrawtransaction.py @@ -28,7 +28,15 @@ class RawTransactionsTest(BitcoinTestFramework): def run_test(self): print "Mining blocks..." - feeTolerance = Decimal(0.00000002) #if the fee's positive delta is higher than this value tests will fail, neg. delta always fail the tests + + min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] + # if the fee's positive delta is higher than this value tests will fail, + # neg. delta always fail the tests. + # The size of the signature of every input may be at most 2 bytes larger + # than a minimum sized signature. + + # = 2 bytes * minRelayTxFeePerByte + feeTolerance = 2 * min_relay_tx_fee/1000 self.nodes[2].generate(1) self.sync_all() diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index b66533543d..7888114c54 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -97,5 +97,19 @@ class HTTPBasicsTest (BitcoinTestFramework): assert_equal('"error":null' in out1, True) assert_equal(conn.sock!=None, True) #connection must be closed because bitcoind should use keep-alive by default + # Check excessive request size + conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port) + conn.connect() + conn.request('GET', '/' + ('x'*1000), '', headers) + out1 = conn.getresponse() + assert_equal(out1.status, httplib.NOT_FOUND) + + conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port) + conn.connect() + conn.request('GET', '/' + ('x'*10000), '', headers) + out1 = conn.getresponse() + assert_equal(out1.status, httplib.BAD_REQUEST) + + if __name__ == '__main__': HTTPBasicsTest ().main () diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index 2824c51ce7..21f8d69382 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -61,6 +61,9 @@ class PruneTest(BitcoinTestFramework): self.address[0] = self.nodes[0].getnewaddress() self.address[1] = self.nodes[1].getnewaddress() + # Determine default relay fee + self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) connect_nodes(self.nodes[2], 0) @@ -239,7 +242,7 @@ class PruneTest(BitcoinTestFramework): outputs = {} t = self.utxo.pop() inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) - remchange = t["amount"] - Decimal("0.001000") + remchange = t["amount"] - 100*self.relayfee # Fee must be above min relay rate for 66kb tx outputs[address]=remchange # Create a basic transaction that will send change back to ourself after account for a fee # And then insert the 128 generated transaction outs in the middle rawtx[92] is where the # diff --git a/src/amount.h b/src/amount.h index 90e6b5aa8e..a4c7764cda 100644 --- a/src/amount.h +++ b/src/amount.h @@ -51,6 +51,7 @@ public: friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; ADD_SERIALIZE_METHODS; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 229dad7b3f..6f22c70494 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -31,9 +31,7 @@ std::string HelpMessageCli() strUsage += HelpMessageOpt("-?", _("This help message")); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), "bitcoin.conf")); strUsage += HelpMessageOpt("-datadir=<dir>", _("Specify data directory")); - strUsage += HelpMessageOpt("-testnet", _("Use the test network")); - strUsage += HelpMessageOpt("-regtest", _("Enter regression test mode, which uses a special chain in which blocks can be " - "solved instantly. This is intended for regression testing tools and app development.")); + AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt("-rpcconnect=<ip>", strprintf(_("Send commands to node running on <ip> (default: %s)"), "127.0.0.1")); strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), 8332, 18332)); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); @@ -94,8 +92,10 @@ static bool AppInitRPC(int argc, char* argv[]) return false; } // Check for -testnet or -regtest parameter (BaseParams() calls are only valid after this clause) - if (!SelectBaseParamsFromCommandLine()) { - fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); + try { + SelectBaseParams(ChainNameFromCommandLine()); + } catch(std::exception &e) { + fprintf(stderr, "Error: %s\n", e.what()); return false; } if (GetBoolArg("-rpcssl", false)) diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index eb36435de6..3330fe5d12 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -35,8 +35,10 @@ static bool AppInitRawTx(int argc, char* argv[]) ParseParameters(argc, argv); // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause) - if (!SelectParamsFromCommandLine()) { - fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); + try { + SelectParams(ChainNameFromCommandLine()); + } catch(std::exception &e) { + fprintf(stderr, "Error: %s\n", e.what()); return false; } @@ -58,8 +60,7 @@ static bool AppInitRawTx(int argc, char* argv[]) strUsage += HelpMessageOpt("-create", _("Create new, empty TX.")); strUsage += HelpMessageOpt("-json", _("Select JSON output")); strUsage += HelpMessageOpt("-txid", _("Output only the hex-encoded transaction id of the resultant transaction.")); - strUsage += HelpMessageOpt("-regtest", _("Enter regression test mode, which uses a special chain in which blocks can be solved instantly.")); - strUsage += HelpMessageOpt("-testnet", _("Use the test network")); + AppendParamsHelpMessages(strUsage); fprintf(stdout, "%s", strUsage.c_str()); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index ca05fefb77..d2af897242 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -107,8 +107,10 @@ bool AppInit(int argc, char* argv[]) return false; } // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause) - if (!SelectParamsFromCommandLine()) { - fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); + try { + SelectParams(ChainNameFromCommandLine()); + } catch(std::exception &e) { + fprintf(stderr, "Error: %s\n", e.what()); return false; } diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 430b75683b..dd26c3b31a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -5,6 +5,7 @@ #include "chainparams.h" +#include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" @@ -76,6 +77,7 @@ public: consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; consensus.fPowAllowMinDifficultyBlocks = false; + consensus.fPowNoRetargeting = false; /** * The message start string is designed to be unlikely to occur in normal data. * The characters are rarely used upper ASCII, not valid as UTF-8, and produce @@ -155,6 +157,7 @@ public: consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; consensus.fPowAllowMinDifficultyBlocks = true; + consensus.fPowNoRetargeting = false; pchMessageStart[0] = 0x0b; pchMessageStart[1] = 0x11; pchMessageStart[2] = 0x09; @@ -217,6 +220,7 @@ public: consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; consensus.fPowAllowMinDifficultyBlocks = true; + consensus.fPowNoRetargeting = true; pchMessageStart[0] = 0xfa; pchMessageStart[1] = 0xbf; @@ -263,31 +267,20 @@ const CChainParams &Params() { return *pCurrentParams; } -CChainParams &Params(CBaseChainParams::Network network) { - switch (network) { - case CBaseChainParams::MAIN: +CChainParams& Params(const std::string& chain) +{ + if (chain == CBaseChainParams::MAIN) return mainParams; - case CBaseChainParams::TESTNET: + else if (chain == CBaseChainParams::TESTNET) return testNetParams; - case CBaseChainParams::REGTEST: + else if (chain == CBaseChainParams::REGTEST) return regTestParams; - default: - assert(false && "Unimplemented network"); - return mainParams; - } + else + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } -void SelectParams(CBaseChainParams::Network network) { +void SelectParams(const std::string& network) +{ SelectBaseParams(network); pCurrentParams = &Params(network); } - -bool SelectParamsFromCommandLine() -{ - CBaseChainParams::Network network = NetworkIdFromCommandLine(); - if (network == CBaseChainParams::MAX_NETWORK_TYPES) - return false; - - SelectParams(network); - return true; -} diff --git a/src/chainparams.h b/src/chainparams.h index 342bccb12f..cb061d596e 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -105,16 +105,15 @@ protected: */ const CChainParams &Params(); -/** Return parameters for the given network. */ -CChainParams &Params(CBaseChainParams::Network network); - -/** Sets the params returned by Params() to those for the given network. */ -void SelectParams(CBaseChainParams::Network network); +/** + * @returns CChainParams for the given BIP70 chain name. + */ +CChainParams& Params(const std::string& chain); /** - * Looks for -regtest or -testnet and then calls SelectParams as appropriate. - * Returns false if an invalid combination is given. + * Sets the params returned by Params() to those for the given BIP70 chain name. + * @throws std::runtime_error when the chain is not supported. */ -bool SelectParamsFromCommandLine(); +void SelectParams(const std::string& chain); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index 9c87bf2154..db2dc751f5 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -5,10 +5,25 @@ #include "chainparamsbase.h" +#include "tinyformat.h" #include "util.h" #include <assert.h> +const std::string CBaseChainParams::MAIN = "main"; +const std::string CBaseChainParams::TESTNET = "test"; +const std::string CBaseChainParams::REGTEST = "regtest"; + +void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp) +{ + strUsage += HelpMessageGroup(_("Chain selection options:")); + strUsage += HelpMessageOpt("-testnet", _("Use the test chain")); + if (debugHelp) { + strUsage += HelpMessageOpt("-regtest", "Enter regression test mode, which uses a special chain in which blocks can be solved instantly. " + "This is intended for regression testing tools and app development."); + } +} + /** * Main network */ @@ -71,31 +86,25 @@ const CBaseChainParams& BaseParams() return *pCurrentBaseParams; } -void SelectBaseParams(CBaseChainParams::Network network) +void SelectBaseParams(const std::string& chain) { - switch (network) { - case CBaseChainParams::MAIN: + if (chain == CBaseChainParams::MAIN) pCurrentBaseParams = &mainParams; - break; - case CBaseChainParams::TESTNET: + else if (chain == CBaseChainParams::TESTNET) pCurrentBaseParams = &testNetParams; - break; - case CBaseChainParams::REGTEST: + else if (chain == CBaseChainParams::REGTEST) pCurrentBaseParams = ®TestParams; - break; - default: - assert(false && "Unimplemented network"); - return; - } + else + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } -CBaseChainParams::Network NetworkIdFromCommandLine() +std::string ChainNameFromCommandLine() { bool fRegTest = GetBoolArg("-regtest", false); bool fTestNet = GetBoolArg("-testnet", false); if (fTestNet && fRegTest) - return CBaseChainParams::MAX_NETWORK_TYPES; + throw std::runtime_error("Invalid combination of -regtest and -testnet."); if (fRegTest) return CBaseChainParams::REGTEST; if (fTestNet) @@ -103,16 +112,6 @@ CBaseChainParams::Network NetworkIdFromCommandLine() return CBaseChainParams::MAIN; } -bool SelectBaseParamsFromCommandLine() -{ - CBaseChainParams::Network network = NetworkIdFromCommandLine(); - if (network == CBaseChainParams::MAX_NETWORK_TYPES) - return false; - - SelectBaseParams(network); - return true; -} - bool AreBaseParamsConfigured() { return pCurrentBaseParams != NULL; diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 4369d0aef7..095c4cbdcb 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -15,13 +15,10 @@ class CBaseChainParams { public: - enum Network { - MAIN, - TESTNET, - REGTEST, - - MAX_NETWORK_TYPES - }; + /** BIP70 chain name strings (main, test or regtest) */ + static const std::string MAIN; + static const std::string TESTNET; + static const std::string REGTEST; const std::string& DataDir() const { return strDataDir; } int RPCPort() const { return nRPCPort; } @@ -34,25 +31,25 @@ protected: }; /** + * Append the help messages for the chainparams options to the + * parameter string. + */ +void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp=true); + +/** * Return the currently selected parameters. This won't change after app * startup, except for unit tests. */ const CBaseChainParams& BaseParams(); /** Sets the params returned by Params() to those for the given network. */ -void SelectBaseParams(CBaseChainParams::Network network); - -/** - * Looks for -regtest or -testnet and returns the appropriate Network ID. - * Returns MAX_NETWORK_TYPES if an invalid combination is given. - */ -CBaseChainParams::Network NetworkIdFromCommandLine(); +void SelectBaseParams(const std::string& chain); /** - * Calls NetworkIdFromCommandLine() and then calls SelectParams as appropriate. - * Returns false if an invalid combination is given. + * Looks for -regtest, -testnet and returns the appropriate BIP70 chain name. + * @return CBaseChainParams::MAX_NETWORK_TYPES if an invalid combination is given. CBaseChainParams::MAIN by default. */ -bool SelectBaseParamsFromCommandLine(); +std::string ChainNameFromCommandLine(); /** * Return true if SelectBaseParamsFromCommandLine() has been called to select diff --git a/src/consensus/params.h b/src/consensus/params.h index c480a1cce1..efbbbed352 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -22,6 +22,7 @@ struct Params { /** Proof of work parameters */ uint256 powLimit; bool fPowAllowMinDifficultyBlocks; + bool fPowNoRetargeting; int64_t nPowTargetSpacing; int64_t nPowTargetTimespan; int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 0a7f903e9f..8698abb900 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -38,6 +38,9 @@ #include <boost/foreach.hpp> #include <boost/scoped_ptr.hpp> +/** Maximum size of http request (request line + headers) */ +static const size_t MAX_HEADERS_SIZE = 8192; + /** HTTP request work item */ class HTTPWorkItem : public HTTPClosure { @@ -414,6 +417,7 @@ bool InitHTTPServer() } evhttp_set_timeout(http, GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); + evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, NULL); diff --git a/src/init.cpp b/src/init.cpp index a079dce5bc..5c961a3ad9 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -320,6 +320,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-dbcache=<n>", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file") + " " + _("on startup")); strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); + strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 @@ -439,11 +441,10 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-printpriority", strprintf("Log transaction priority and fee per kB when mining blocks (default: %u)", 0)); strUsage += HelpMessageOpt("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", 1)); - strUsage += HelpMessageOpt("-regtest", "Enter regression test mode, which uses a special chain in which blocks can be solved instantly. " - "This is intended for regression testing tools and app development."); } strUsage += HelpMessageOpt("-shrinkdebugfile", _("Shrink debug.log file on client startup (default: 1 when no -debug)")); - strUsage += HelpMessageOpt("-testnet", _("Use the test network")); + + AppendParamsHelpMessages(strUsage, showDebug); strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) @@ -841,6 +842,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) fCheckBlockIndex = GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); fCheckpointsEnabled = GetBoolArg("-checkpoints", true); + // -mempoollimit limits + int64_t nMempoolSizeLimit = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolDescendantSizeLimit = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + if (nMempoolSizeLimit < 0 || nMempoolSizeLimit < nMempoolDescendantSizeLimit * 40) + return InitError(strprintf(_("Error: -maxmempool must be at least %d MB"), GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) / 25)); + // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency nScriptCheckThreads = GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (nScriptCheckThreads <= 0) diff --git a/src/main.cpp b/src/main.cpp index baad7fc050..d870280e99 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -75,7 +75,7 @@ uint64_t nPruneTarget = 0; bool fAlerts = DEFAULT_ALERTS; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ -CFeeRate minRelayTxFee = CFeeRate(5000); +CFeeRate minRelayTxFee = CFeeRate(1000); CTxMemPool mempool(::minRelayTxFee); @@ -740,17 +740,14 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) return true; } -CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree) +CAmount GetMinRelayFee(const CTransaction& tx, const CTxMemPool& pool, unsigned int nBytes, bool fAllowFree) { - { - LOCK(mempool.cs); - uint256 hash = tx.GetHash(); - double dPriorityDelta = 0; - CAmount nFeeDelta = 0; - mempool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); - if (dPriorityDelta > 0 || nFeeDelta > 0) - return 0; - } + uint256 hash = tx.GetHash(); + double dPriorityDelta = 0; + CAmount nFeeDelta = 0; + pool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); + if (dPriorityDelta > 0 || nFeeDelta > 0) + return 0; CAmount nMinFee = ::minRelayTxFee.GetFee(nBytes); @@ -779,7 +776,7 @@ static std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectAbsurdFee) + bool* pfMissingInputs, bool fOverrideMempoolLimit, bool fRejectAbsurdFee) { AssertLockHeld(cs_main); if (pfMissingInputs) @@ -879,17 +876,20 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CAmount nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx)); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx)); unsigned int nSize = entry.GetTxSize(); // Don't accept it if it can't get into a block - CAmount txMinFee = GetMinRelayFee(tx, nSize, true); + CAmount txMinFee = GetMinRelayFee(tx, pool, nSize, true); if (fLimitFree && nFees < txMinFee) return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient fee", false, strprintf("%d < %d", nFees, txMinFee)); - // Require that free transactions have sufficient priority to be mined in the next block. - if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) { + CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); + if (mempoolRejectFee > 0 && nFees < mempoolRejectFee) { + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); + } else if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) { + // Require that free transactions have sufficient priority to be mined in the next block. return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); } @@ -954,6 +954,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + + // trim mempool and check if tx was trimmed + if (!fOverrideMempoolLimit) { + int expired = pool.Expire(GetTime() - GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + if (expired != 0) + LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); + + pool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (!pool.exists(tx.GetHash())) + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); + } } SyncWithWallets(tx, NULL); @@ -2020,7 +2031,7 @@ void static UpdateTip(CBlockIndex *pindexNew) { } } -/** Disconnect chainActive's tip. */ +/** Disconnect chainActive's tip. You want to manually re-limit mempool size after this */ bool static DisconnectTip(CValidationState &state) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2047,7 +2058,7 @@ bool static DisconnectTip(CValidationState &state) { // ignore validation errors in resurrected transactions list<CTransaction> removed; CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) { + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { mempool.remove(tx, removed, true); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); @@ -2220,9 +2231,11 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. + bool fBlocksDisconnected = false; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { if (!DisconnectTip(state)) return false; + fBlocksDisconnected = true; } // Build list of new blocks to connect. @@ -2268,6 +2281,9 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo } } + if (fBlocksDisconnected) + mempool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + // Callbacks/notifications for a new best chain. if (fInvalidFound) CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); @@ -2354,6 +2370,8 @@ bool InvalidateBlock(CValidationState& state, CBlockIndex *pindex) { } } + mempool.TrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. BlockMap::iterator it = mapBlockIndex.begin(); @@ -4290,10 +4308,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, RelayTransaction(tx); vWorkQueue.push_back(inv.hash); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u)\n", + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom->id, tx.GetHash().ToString(), - mempool.size()); + mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; @@ -4955,7 +4973,16 @@ bool SendMessages(CNode* pto, bool fSendTrickle) if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; nSyncStarted++; - CBlockIndex *pindexStart = pindexBestHeader->pprev ? pindexBestHeader->pprev : pindexBestHeader; + const CBlockIndex *pindexStart = pindexBestHeader; + /* If possible, start at the block preceding the currently + best known header. This ensures that we always get a + non-empty list of headers back as long as the peer + is up-to-date. With a non-empty response, we can initialise + the peer's known best block. This wouldn't be possible + if we requested starting at pindexBestHeader and + got back an empty response. */ + if (pindexStart->pprev) + pindexStart = pindexStart->pprev; LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); pto->PushMessage("getheaders", chainActive.GetLocator(pindexStart), uint256()); } diff --git a/src/main.h b/src/main.h index a6001eed8f..202d2c772b 100644 --- a/src/main.h +++ b/src/main.h @@ -51,6 +51,10 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500; +/** Default for -maxmempool, maximum megabytes of mempool memory usage */ +static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300; +/** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ +static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 72; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -225,7 +229,7 @@ void PruneAndFlush(); /** (try to) add transaction to memory pool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectAbsurdFee=false); + bool* pfMissingInputs, bool fOverrideMempoolLimit=false, bool fRejectAbsurdFee=false); struct CNodeStateStats { diff --git a/src/pow.cpp b/src/pow.cpp index bb53ad204b..5ace3fbc9b 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -52,6 +52,9 @@ unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHead unsigned int CalculateNextWorkRequired(const CBlockIndex* pindexLast, int64_t nFirstBlockTime, const Consensus::Params& params) { + if (params.fPowNoRetargeting) + return pindexLast->nBits; + // Limit adjustment step int64_t nActualTimespan = pindexLast->GetBlockTime() - nFirstBlockTime; LogPrintf(" nActualTimespan = %d before bounds\n", nActualTimespan); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 86c820cd0f..bda8acff15 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -597,8 +597,10 @@ int main(int argc, char *argv[]) // - Needs to be done before createOptionsModel // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause) - if (!SelectParamsFromCommandLine()) { - QMessageBox::critical(0, QObject::tr("Bitcoin Core"), QObject::tr("Error: Invalid combination of -regtest and -testnet.")); + try { + SelectParams(ChainNameFromCommandLine()); + } catch(std::exception &e) { + QMessageBox::critical(0, QObject::tr("Bitcoin Core"), QObject::tr("Error: %1").arg(e.what())); return 1; } #ifdef ENABLE_WALLET diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 4dec53396d..4b96473504 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -809,7 +809,7 @@ UniValue sendrawtransaction(const UniValue& params, bool fHelp) // push to local node and sync with wallets CValidationState state; bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, !fOverrideFees)) { + if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, false, !fOverrideFees)) { if (state.IsInvalid()) { throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); } else { diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 5bf1e98e8f..0cf906a259 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -153,11 +153,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> sortedOrder; sortedOrder.resize(5); - sortedOrder[0] = tx2.GetHash().ToString(); // 20000 - sortedOrder[1] = tx4.GetHash().ToString(); // 15000 + sortedOrder[0] = tx3.GetHash().ToString(); // 0 + sortedOrder[1] = tx5.GetHash().ToString(); // 10000 sortedOrder[2] = tx1.GetHash().ToString(); // 10000 - sortedOrder[3] = tx5.GetHash().ToString(); // 10000 - sortedOrder[4] = tx3.GetHash().ToString(); // 0 + sortedOrder[3] = tx4.GetHash().ToString(); // 15000 + sortedOrder[4] = tx2.GetHash().ToString(); // 20000 CheckSort(pool, sortedOrder); /* low fee but with high fee child */ @@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.addUnchecked(tx6.GetHash(), CTxMemPoolEntry(tx6, 0LL, 1, 10.0, 1, true)); BOOST_CHECK_EQUAL(pool.size(), 6); // Check that at this point, tx6 is sorted low - sortedOrder.push_back(tx6.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); CheckSort(pool, sortedOrder); CTxMemPool::setEntries setAncestors; @@ -194,9 +194,9 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 7); // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... - sortedOrder.erase(sortedOrder.end()-1); - sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); - sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); + sortedOrder.erase(sortedOrder.begin()); + sortedOrder.push_back(tx6.GetHash().ToString()); + sortedOrder.push_back(tx7.GetHash().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.addUnchecked(tx8.GetHash(), CTxMemPoolEntry(tx8, 0LL, 2, 10.0, 1, true), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high - sortedOrder.push_back(tx8.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx8.GetHash().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) // tx9 should be sorted low BOOST_CHECK_EQUAL(pool.size(), 9); - sortedOrder.push_back(tx9.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx9.GetHash().ToString()); CheckSort(pool, sortedOrder); std::vector<std::string> snapshotOrder = sortedOrder; @@ -255,21 +255,21 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) * tx8 and tx9 should both now be sorted higher * Final order after tx10 is added: * - * tx7 = 2.2M (4 txs) - * tx6 = 2.2M (5 txs) - * tx10 = 200k (1 tx) - * tx8 = 200k (2 txs) - * tx9 = 200k (2 txs) - * tx2 = 20000 (1) - * tx4 = 15000 (1) - * tx1 = 10000 (1) - * tx5 = 10000 (1) * tx3 = 0 (1) + * tx5 = 10000 (1) + * tx1 = 10000 (1) + * tx4 = 15000 (1) + * tx2 = 20000 (1) + * tx9 = 200k (2 txs) + * tx8 = 200k (2 txs) + * tx10 = 200k (1 tx) + * tx6 = 2.2M (5 txs) + * tx7 = 2.2M (4 txs) */ - sortedOrder.erase(sortedOrder.end()-2, sortedOrder.end()); // take out tx8, tx9 from the end - sortedOrder.insert(sortedOrder.begin()+2, tx10.GetHash().ToString()); // tx10 is after tx6 - sortedOrder.insert(sortedOrder.begin()+3, tx9.GetHash().ToString()); - sortedOrder.insert(sortedOrder.begin()+3, tx8.GetHash().ToString()); + sortedOrder.erase(sortedOrder.begin(), sortedOrder.begin()+2); // take out tx9, tx8 from the beginning + sortedOrder.insert(sortedOrder.begin()+5, tx9.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin()+6, tx8.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin()+7, tx10.GetHash().ToString()); // tx10 is just before tx6 CheckSort(pool, sortedOrder); // there should be 10 transactions in the mempool @@ -281,4 +281,157 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort(pool, snapshotOrder); } +BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) +{ + CTxMemPool pool(CFeeRate(1000)); + + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vin.resize(1); + tx1.vin[0].scriptSig = CScript() << OP_1; + tx1.vout.resize(1); + tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx1.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx1.GetHash(), CTxMemPoolEntry(tx1, 10000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx1))); + + CMutableTransaction tx2 = CMutableTransaction(); + tx2.vin.resize(1); + tx2.vin[0].scriptSig = CScript() << OP_2; + tx2.vout.resize(1); + tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; + tx2.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx2.GetHash(), CTxMemPoolEntry(tx2, 5000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx2))); + + pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing + BOOST_CHECK(pool.exists(tx1.GetHash())); + BOOST_CHECK(pool.exists(tx2.GetHash())); + + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // should remove the lower-feerate transaction + BOOST_CHECK(pool.exists(tx1.GetHash())); + BOOST_CHECK(!pool.exists(tx2.GetHash())); + + pool.addUnchecked(tx2.GetHash(), CTxMemPoolEntry(tx2, 5000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx2))); + CMutableTransaction tx3 = CMutableTransaction(); + tx3.vin.resize(1); + tx3.vin[0].prevout = COutPoint(tx2.GetHash(), 0); + tx3.vin[0].scriptSig = CScript() << OP_2; + tx3.vout.resize(1); + tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; + tx3.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx3.GetHash(), CTxMemPoolEntry(tx3, 20000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx3))); + + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) + BOOST_CHECK(!pool.exists(tx1.GetHash())); + BOOST_CHECK(pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(tx3.GetHash())); + + pool.TrimToSize(::GetSerializeSize(CTransaction(tx1), SER_NETWORK, PROTOCOL_VERSION)); // mempool is limited to tx1's size in memory usage, so nothing fits + BOOST_CHECK(!pool.exists(tx1.GetHash())); + BOOST_CHECK(!pool.exists(tx2.GetHash())); + BOOST_CHECK(!pool.exists(tx3.GetHash())); + + CFeeRate maxFeeRateRemoved(25000, ::GetSerializeSize(CTransaction(tx3), SER_NETWORK, PROTOCOL_VERSION) + ::GetSerializeSize(CTransaction(tx2), SER_NETWORK, PROTOCOL_VERSION)); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); + + CMutableTransaction tx4 = CMutableTransaction(); + tx4.vin.resize(2); + tx4.vin[0].prevout.SetNull(); + tx4.vin[0].scriptSig = CScript() << OP_4; + tx4.vin[1].prevout.SetNull(); + tx4.vin[1].scriptSig = CScript() << OP_4; + tx4.vout.resize(2); + tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; + tx4.vout[0].nValue = 10 * COIN; + tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; + tx4.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx5 = CMutableTransaction(); + tx5.vin.resize(2); + tx5.vin[0].prevout = COutPoint(tx4.GetHash(), 0); + tx5.vin[0].scriptSig = CScript() << OP_4; + tx5.vin[1].prevout.SetNull(); + tx5.vin[1].scriptSig = CScript() << OP_5; + tx5.vout.resize(2); + tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; + tx5.vout[0].nValue = 10 * COIN; + tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; + tx5.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx6 = CMutableTransaction(); + tx6.vin.resize(2); + tx6.vin[0].prevout = COutPoint(tx4.GetHash(), 1); + tx6.vin[0].scriptSig = CScript() << OP_4; + tx6.vin[1].prevout.SetNull(); + tx6.vin[1].scriptSig = CScript() << OP_6; + tx6.vout.resize(2); + tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; + tx6.vout[0].nValue = 10 * COIN; + tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; + tx6.vout[1].nValue = 10 * COIN; + + CMutableTransaction tx7 = CMutableTransaction(); + tx7.vin.resize(2); + tx7.vin[0].prevout = COutPoint(tx5.GetHash(), 0); + tx7.vin[0].scriptSig = CScript() << OP_5; + tx7.vin[1].prevout = COutPoint(tx6.GetHash(), 0); + tx7.vin[1].scriptSig = CScript() << OP_6; + tx7.vout.resize(2); + tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + + pool.addUnchecked(tx4.GetHash(), CTxMemPoolEntry(tx4, 7000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx4))); + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx6.GetHash(), CTxMemPoolEntry(tx6, 1100LL, 0, 10.0, 1, pool.HasNoInputsOf(tx6))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + // we only require this remove, at max, 2 txn, because its not clear what we're really optimizing for aside from that + pool.TrimToSize(pool.DynamicMemoryUsage() - 1); + BOOST_CHECK(pool.exists(tx4.GetHash())); + BOOST_CHECK(pool.exists(tx6.GetHash())); + BOOST_CHECK(!pool.exists(tx7.GetHash())); + + if (!pool.exists(tx5.GetHash())) + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 + BOOST_CHECK(pool.exists(tx4.GetHash())); + BOOST_CHECK(!pool.exists(tx5.GetHash())); + BOOST_CHECK(pool.exists(tx6.GetHash())); + BOOST_CHECK(!pool.exists(tx7.GetHash())); + + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 1000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx5))); + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 9000LL, 0, 10.0, 1, pool.HasNoInputsOf(tx7))); + + std::vector<CTransaction> vtx; + std::list<CTransaction> conflicts; + SetMockTime(42); + SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); + // ... we should keep the same min fee until we get a block + pool.removeForBlock(vtx, 1, conflicts); + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/2); + // ... then feerate should drop 1/2 each halflife + + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2); + BOOST_CHECK_EQUAL(pool.GetMinFee(pool.DynamicMemoryUsage() * 5 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/4); + // ... with a 1/2 halflife when mempool is < 1/2 its target size + + SetMockTime(42 + 2*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(pool.DynamicMemoryUsage() * 9 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + 1000)/8); + // ... with a 1/4 halflife when mempool is < 1/4 its target size + + SetMockTime(42 + 7*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), 1000); + // ... but feerate should never drop below 1000 + + SetMockTime(42 + 8*CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE/2 + CTxMemPool::ROLLING_FEE_HALFLIFE/4); + BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), 0); + // ... unless it has gone all the way to 0 (after getting past 1000/2) + + SetMockTime(0); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 8d81275a6f..a74fbfc0d7 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -32,13 +32,13 @@ CWallet* pwalletMain; extern bool fPrintToConsole; extern void noui_connect(); -BasicTestingSetup::BasicTestingSetup(CBaseChainParams::Network network) +BasicTestingSetup::BasicTestingSetup(const std::string& chainName) { ECC_Start(); SetupEnvironment(); fPrintToDebugLog = false; // don't want to write to debug.log file fCheckBlockIndex = true; - SelectParams(network); + SelectParams(chainName); noui_connect(); } @@ -47,7 +47,7 @@ BasicTestingSetup::~BasicTestingSetup() ECC_Stop(); } -TestingSetup::TestingSetup(CBaseChainParams::Network network) : BasicTestingSetup(network) +TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { #ifdef ENABLE_WALLET bitdb.MakeMock(); diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index b9314d0611..0bab4b6831 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -12,7 +12,7 @@ * This just configures logging and chain parameters. */ struct BasicTestingSetup { - BasicTestingSetup(CBaseChainParams::Network network = CBaseChainParams::MAIN); + BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~BasicTestingSetup(); }; @@ -25,7 +25,7 @@ struct TestingSetup: public BasicTestingSetup { boost::filesystem::path pathTemp; boost::thread_group threadGroup; - TestingSetup(CBaseChainParams::Network network = CBaseChainParams::MAIN); + TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~TestingSetup(); }; diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index edad18644e..9b8e1c088b 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx) LOCK(cs_main); CValidationState state; - return AcceptToMemoryPool(mempool, state, tx, false, NULL, false); + return AcceptToMemoryPool(mempool, state, tx, false, NULL, true, false); } BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 1370cab0c0..bb148005cd 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -13,6 +13,7 @@ #include "streams.h" #include "util.h" #include "utilmoneystr.h" +#include "utiltime.h" #include "version.h" using namespace std; @@ -305,15 +306,18 @@ void CTxMemPoolEntry::UpdateState(int64_t modifySize, CAmount modifyFee, int64_t } } -CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : +CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : nTransactionsUpdated(0) { + clear(); + // Sanity checks off by default for performance, because otherwise // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool fSanityCheck = false; - minerPolicyEstimator = new CBlockPolicyEstimator(_minRelayFee); + minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); + minReasonableRelayFee = _minReasonableRelayFee; } CTxMemPool::~CTxMemPool() @@ -538,6 +542,8 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i } // After the txs in the new block have been removed from the mempool, update policy estimates minerPolicyEstimator->processBlock(nBlockHeight, entries, fCurrentEstimate); + lastRollingFeeUpdate = GetTime(); + blockSinceLastRollingFeeBump = true; } void CTxMemPool::clear() @@ -548,6 +554,9 @@ void CTxMemPool::clear() mapNextTx.clear(); totalTxSize = 0; cachedInnerUsage = 0; + lastRollingFeeUpdate = GetTime(); + blockSinceLastRollingFeeBump = false; + rollingMinimumFeeRate = 0; ++nTransactionsUpdated; } @@ -735,10 +744,10 @@ void CTxMemPool::PrioritiseTransaction(const uint256 hash, const string strHash, LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) +void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const { LOCK(cs); - std::map<uint256, std::pair<double, CAmount> >::iterator pos = mapDeltas.find(hash); + std::map<uint256, std::pair<double, CAmount> >::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; const std::pair<double, CAmount> &deltas = pos->second; @@ -792,6 +801,22 @@ void CTxMemPool::RemoveStaged(setEntries &stage) { } } +int CTxMemPool::Expire(int64_t time) { + LOCK(cs); + indexed_transaction_set::nth_index<2>::type::iterator it = mapTx.get<2>().begin(); + setEntries toremove; + while (it != mapTx.get<2>().end() && it->GetTime() < time) { + toremove.insert(mapTx.project<0>(it)); + it++; + } + setEntries stage; + BOOST_FOREACH(txiter removeit, toremove) { + CalculateDescendants(removeit, stage); + } + RemoveStaged(stage); + return stage.size(); +} + bool CTxMemPool::addUnchecked(const uint256&hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) { LOCK(cs); @@ -837,3 +862,62 @@ const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) cons assert(it != mapLinks.end()); return it->second.children; } + +CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { + LOCK(cs); + if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) + return CFeeRate(rollingMinimumFeeRate); + + int64_t time = GetTime(); + if (time > lastRollingFeeUpdate + 10) { + double halflife = ROLLING_FEE_HALFLIFE; + if (DynamicMemoryUsage() < sizelimit / 4) + halflife /= 4; + else if (DynamicMemoryUsage() < sizelimit / 2) + halflife /= 2; + + rollingMinimumFeeRate = rollingMinimumFeeRate / pow(2.0, (time - lastRollingFeeUpdate) / halflife); + lastRollingFeeUpdate = time; + + if (rollingMinimumFeeRate < minReasonableRelayFee.GetFeePerK() / 2) { + rollingMinimumFeeRate = 0; + return CFeeRate(0); + } + } + return std::max(CFeeRate(rollingMinimumFeeRate), minReasonableRelayFee); +} + +void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { + AssertLockHeld(cs); + if (rate.GetFeePerK() > rollingMinimumFeeRate) { + rollingMinimumFeeRate = rate.GetFeePerK(); + blockSinceLastRollingFeeBump = false; + } +} + +void CTxMemPool::TrimToSize(size_t sizelimit) { + LOCK(cs); + + unsigned nTxnRemoved = 0; + CFeeRate maxFeeRateRemoved(0); + while (DynamicMemoryUsage() > sizelimit) { + indexed_transaction_set::nth_index<1>::type::iterator it = mapTx.get<1>().begin(); + + // We set the new mempool min fee to the feerate of the removed set, plus the + // "minimum reasonable fee rate" (ie some value under which we consider txn + // to have 0 fee). This way, we don't allow txn to enter mempool with feerate + // equal to txn which were removed with no block in between. + CFeeRate removed(it->GetFeesWithDescendants(), it->GetSizeWithDescendants()); + removed += minReasonableRelayFee; + trackPackageRemoved(removed); + maxFeeRateRemoved = std::max(maxFeeRateRemoved, removed); + + setEntries stage; + CalculateDescendants(mapTx.project<0>(it), stage); + RemoveStaged(stage); + nTxnRemoved += stage.size(); + } + + if (maxFeeRateRemoved > CFeeRate(0)) + LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); +} diff --git a/src/txmempool.h b/src/txmempool.h index c0eef0dd22..d44995eefe 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -83,7 +83,7 @@ public: const CTransaction& GetTx() const { return this->tx; } double GetPriority(unsigned int currentHeight) const; - CAmount GetFee() const { return nFee; } + const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return nHeight; } @@ -160,9 +160,9 @@ public: double f2 = aSize * bFees; if (f1 == f2) { - return a.GetTime() < b.GetTime(); + return a.GetTime() >= b.GetTime(); } - return f1 > f2; + return f1 < f2; } // Calculate which feerate to use for an entry (avoiding division). @@ -211,9 +211,10 @@ public: * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * - * mapTx is a boost::multi_index that sorts the mempool on 2 criteria: + * mapTx is a boost::multi_index that sorts the mempool on 3 criteria: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] + * - time in mempool * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given @@ -284,7 +285,18 @@ private: uint64_t totalTxSize; //! sum of all mempool tx' byte sizes uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves) + CFeeRate minReasonableRelayFee; + + mutable int64_t lastRollingFeeUpdate; + mutable bool blockSinceLastRollingFeeBump; + mutable double rollingMinimumFeeRate; //! minimum fee to get into the pool, decreases exponentially + + void trackPackageRemoved(const CFeeRate& rate); + public: + + static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; // public only for testing + typedef boost::multi_index_container< CTxMemPoolEntry, boost::multi_index::indexed_by< @@ -294,6 +306,11 @@ public: boost::multi_index::ordered_non_unique< boost::multi_index::identity<CTxMemPoolEntry>, CompareTxMemPoolEntryByFee + >, + // sorted by entry time + boost::multi_index::ordered_non_unique< + boost::multi_index::identity<CTxMemPoolEntry>, + CompareTxMemPoolEntryByEntryTime > > > indexed_transaction_set; @@ -328,7 +345,12 @@ public: std::map<COutPoint, CInPoint> mapNextTx; std::map<uint256, std::pair<double, CAmount> > mapDeltas; - CTxMemPool(const CFeeRate& _minRelayFee); + /** Create a new CTxMemPool. + * minReasonableRelayFee should be a feerate which is, roughly, somewhere + * around what it "costs" to relay a transaction around the network and + * below which we would reasonably say a transaction has 0-effective-fee. + */ + CTxMemPool(const CFeeRate& _minReasonableRelayFee); ~CTxMemPool(); /** @@ -365,7 +387,7 @@ public: /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); - void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta); + void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: @@ -397,6 +419,20 @@ public: */ bool CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true); + /** The minimum fee to get into the mempool, which may itself not be enough + * for larger-sized transactions. + * The minReasonableRelayFee constructor arg is used to bound the time it + * takes the fee rate to go back down all the way to 0. When the feerate + * would otherwise be half of this, it is set to 0 instead. + */ + CFeeRate GetMinFee(size_t sizelimit) const; + + /** Remove transactions from the mempool until its dynamic size is <= sizelimit. */ + void TrimToSize(size_t sizelimit); + + /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ + int Expire(int64_t time); + unsigned long size() { LOCK(cs); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index bd3004061b..3f2d5a05f6 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2863,6 +2863,6 @@ int CMerkleTx::GetBlocksToMaturity() const bool CMerkleTx::AcceptToMemoryPool(bool fLimitFree, bool fRejectAbsurdFee) { CValidationState state; - return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, fRejectAbsurdFee); + return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, false, fRejectAbsurdFee); } |