diff options
36 files changed, 1597 insertions, 879 deletions
diff --git a/.travis.yml b/.travis.yml index 6e3fc5144c..69397c26bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ cache: - depends/built - depends/sdk-sources - $HOME/.ccache -git: - depth: 1 env: global: - MAKEJOBS=-j3 diff --git a/configure.ac b/configure.ac index 4f57750a1a..c422914a26 100644 --- a/configure.ac +++ b/configure.ac @@ -219,6 +219,12 @@ AC_ARG_ENABLE([debug], [enable_debug=$enableval], [enable_debug=no]) +# Enable different -fsanitize options +AC_ARG_WITH([sanitizers], + [AS_HELP_STRING([--with-sanitizers], + [comma separated list of extra sanitizers to build with (default is none enabled)])], + [use_sanitizers=$withval]) + # Enable gprof profiling AC_ARG_ENABLE([gprof], [AS_HELP_STRING([--enable-gprof], @@ -247,6 +253,26 @@ if test "x$enable_debug" = xyes; then fi fi +if test x$use_sanitizers != x; then + # First check if the compiler accepts flags. If an incompatible pair like + # -fsanitize=address,thread is used here, this check will fail. This will also + # fail if a bad argument is passed, e.g. -fsanitize=undfeined + AX_CHECK_COMPILE_FLAG( + [[-fsanitize=$use_sanitizers]], + [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], + [AC_MSG_ERROR([compiler did not accept requested flags])]) + + # Some compilers (e.g. GCC) require additional libraries like libasan, + # libtsan, libubsan, etc. Make sure linking still works with the sanitize + # flag. This is a separate check so we can give a better error message when + # the sanitize flags are supported by the compiler but the actual sanitizer + # libs are missing. + AX_CHECK_LINK_FLAG( + [[-fsanitize=$use_sanitizers]], + [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], + [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])]) +fi + ERROR_CXXFLAGS= if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then @@ -1258,6 +1284,8 @@ AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) +AC_SUBST(SANITIZER_CXXFLAGS) +AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_UPNP) diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index ca1bd8854f..5402870fba 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -71,7 +71,6 @@ import argparse import difflib import io import re -import string import subprocess import sys diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index b8f54192c9..0f2099953f 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -20,38 +20,38 @@ def check_ELF_PIE(executable): ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): line = line.split() - if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN': + if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': ok = True return ok def get_ELF_program_headers(executable): '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') in_headers = False count = 0 headers = [] - for line in stdout.split(b'\n'): - if line.startswith(b'Program Headers:'): + for line in stdout.splitlines(): + if line.startswith('Program Headers:'): in_headers = True - if line == b'': + if line == '': in_headers = False if in_headers: if count == 1: # header line - ofs_typ = line.find(b'Type') - ofs_offset = line.find(b'Offset') - ofs_flags = line.find(b'Flg') - ofs_align = line.find(b'Align') + ofs_typ = line.find('Type') + ofs_offset = line.find('Offset') + ofs_flags = line.find('Flg') + ofs_align = line.find('Align') if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: raise ValueError('Cannot parse elfread -lW output') elif count > 1: @@ -68,9 +68,9 @@ def check_ELF_NX(executable): have_wx = False have_gnu_stack = False for (typ, flags) in get_ELF_program_headers(executable): - if typ == b'GNU_STACK': + if typ == 'GNU_STACK': have_gnu_stack = True - if b'W' in flags and b'E' in flags: # section is both writable and executable + if 'W' in flags and 'E' in flags: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx @@ -87,17 +87,17 @@ def check_ELF_RELRO(executable): # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 - if typ == b'GNU_RELRO': + if typ == 'GNU_RELRO': have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]): + if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]): have_bindnow = True return have_gnu_relro and have_bindnow @@ -105,13 +105,13 @@ def check_ELF_Canary(executable): ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False - for line in stdout.split(b'\n'): - if b'__stack_chk_fail' in line: + for line in stdout.splitlines(): + if '__stack_chk_fail' in line: ok = True return ok @@ -121,13 +121,13 @@ def get_PE_dll_characteristics(executable): Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' and bits is the DllCharacteristics value. ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') arch = '' bits = 0 - for line in stdout.split('\n'): + for line in stdout.splitlines(): tokens = line.split() if len(tokens)>=2 and tokens[0] == 'architecture:': arch = tokens[1].rstrip(',') diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 2b77857766..3a67319eaa 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -46,28 +46,28 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -b'_edata', b'_end', b'_init', b'__bss_start', b'_fini', b'_IO_stdin_used' +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -b'libgcc_s.so.1', # GCC base support -b'libc.so.6', # C library -b'libpthread.so.0', # threading -b'libanl.so.1', # DNS resolve -b'libm.so.6', # math library -b'librt.so.1', # real-time (clock) -b'ld-linux-x86-64.so.2', # 64-bit dynamic linker -b'ld-linux.so.2', # 32-bit dynamic linker +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker # bitcoin-qt only -b'libX11-xcb.so.1', # part of X11 -b'libX11.so.6', # part of X11 -b'libxcb.so.1', # part of X11 -b'libfontconfig.so.1', # font support -b'libfreetype.so.6', # font parsing -b'libdl.so.2' # programming interface to dynamic linker +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker } class CPPFilt(object): @@ -77,10 +77,10 @@ class CPPFilt(object): Use a pipe to the 'c++filt' command. ''' def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): - self.proc.stdin.write(mangled + b'\n') + self.proc.stdin.write(mangled + '\n') self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() @@ -94,43 +94,43 @@ def read_symbols(executable, imports=True): Parse an ELF executable and return a list of (symbol,version) tuples for dynamic, imported symbols. ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) syms = [] - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): line = line.split() - if len(line)>7 and re.match(b'[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition(b'@') - is_import = line[6] == b'UND' - if version.startswith(b'@'): + if len(line)>7 and re.match('[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition('@') + is_import = line[6] == 'UND' + if version.startswith('@'): version = version[1:] if is_import == imports: syms.append((sym, version)) return syms def check_version(max_versions, version): - if b'_' in version: - (lib, _, ver) = version.rpartition(b'_') + if '_' in version: + (lib, _, ver) = version.rpartition('_') else: lib = version ver = '0' - ver = tuple([int(x) for x in ver.split(b'.')]) + ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False return ver <= max_versions[lib] def read_libraries(filename): - p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') libraries = [] - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>2 and tokens[1] == b'(NEEDED)': - match = re.match(b'^Shared library: \[(.*)\]$', b' '.join(tokens[2:])) + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: @@ -144,18 +144,18 @@ if __name__ == '__main__': # Check imported symbols for sym,version in read_symbols(filename, True): if version and not check_version(MAX_VERSIONS, version): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym).decode('utf-8'), version.decode('utf-8'))) + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) retval = 1 # Check exported symbols for sym,version in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym).decode('utf-8'))) + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name.decode('utf-8'))) + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) retval = 1 sys.exit(retval) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index e9e3f65da2..fa492fdd27 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -24,3 +24,24 @@ keys: Note that the above isn't a good UI/UX yet, and needs significant improvements to make it more convenient and reduce the chance of errors; pull-reqs improving this process would be much appreciated. + +Configuration files +------------------- + +* `trusted-git-root`: This file should contain a single git commit hash which is the first unsigned git commit (hence it is the "root of trust"). +* `trusted-sha512-root-commit`: This file should contain a single git commit hash which is the first commit without a SHA512 root commitment. +* `trusted-keys`: This file should contain a \n-delimited list of all PGP fingerprints of authorized commit signers (primary, not subkeys). +* `allow-revsig-commits`: This file should contain a \n-delimited list of git commit hashes. See next section for more info. + +Key expiry/revocation +--------------------- + +When a key (or subkey) which has signed old commits expires or is revoked, +verify-commits will start failing to verify all commits which were signed by +said key. In order to avoid bumping the root-of-trust `trusted-git-root` +file, individual commits which were signed by such a key can be added to the +`allow-revsig-commits` file. That way, the PGP signatures are still verified +but no new commits can be signed by any expired/revoked key. To easily build a +list of commits which need to be added, verify-commits.sh can be edited to test +each commit with BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG set to both 1 and 0, and +those which need it set to 1 printed. diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index f0088cdca4..3abf82e529 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -102,3 +102,403 @@ bafd075c5e6a1088ef0f1aa0b0b224e026a3d3e0 c8d2473e6cb042e7275a10c49d3f6a4a91bf0166 386f4385ab04b0b2c3d47bddc0dc0f2de7354964 9f33dba05c01ecc5c56eb1284ab7d64d42f55171 +7466a26cab5d66665991433947964a638f5b957e +b43aba89e356ff95b706e80d4802f60fc46a569a +02b7e8319aef2a870264ad4fa2e3bb18664dcc36 +f686002a8eba820a40ac2f34a6e8f57b2b5cc54c +2b1c50b9352ab1dc40b0f877db23c1fa4048fae3 +2405ce1df043f778b8efb9205009500cbc17313a +4ad3b3c72c73d61e0a0cab541dca20acf651320d +4ba3d4f4393d81148422d24d222fe7ed00130194 +8ee5c7b747171e335793c74cd9d2f7491da58164 +872c921c0a208b04bd0713758e52fcab5b7c1684 +00d1680498c5550e7db1f359202d3433a092fafd +585db41e9ab7a6fb262c8bad7f427cdbdc497188 +18462960c0f13bd07d8f52b61e7d7bc17e991eea +0630974647dacaf25e7fcb7f9cbb785bb078ede6 +0f58d7f3d62f012f2584f5e781fc73de4763dd9e +3d16f581538b0974853e820508e8b3093269d2fd +66e91420ab233cf1dac64504e0dc129019bf8c0d +d8d9162f5bad39b2720dd2b2da237c6159e4755f +29fad97c320c892ab6a480c81e2078ec22ab354b +791c3ea61b4e49fd46a1a71b84ca99ddf69d2ff7 +a312e201ba56742499a5480b5f2115f01505c217 +ce56fdd2e8cdf94fd0ab76d71adbfa755e23ce7d +480f42630cbd598c04fa59ee0e406f56904ecffb +6012f1caf744ac9b53383d7d10a8f1b70ca2c0e1 +ded6a2afa549f693dcabb430ce0862f8631360c8 +07090c5339436f856e79a8036d1c85deeb453803 +0e265916d1c6a63e4a3821dab9db597b5ec64b46 +e4ffcacc2187d3419c8ea12b82fb06d82d8751d2 +e117cfe45eee9169409e74a44ef4a866be25bc35 +dcfe218626b05204e9fbc95ba5d95ca0eb72ec9b +23481fa50301201ef5a60675ef899aa6ce94ca03 +27c59dc502f29cf1d76290556c21e366145e3b2e +4a62ddd01873d18dbca96c81d756be1020249b45 +a233fb4f1d037e68ff70eef3a9f5b7bf1d631918 +b2089c51cc4af2f7e1c0ec75be9449ee222b1d69 +c997f8808256521397f1c003bb1e9896fee6eaa0 +5dc00f68c49c46a380a98d06233f90528b8e2557 +fe53d5f3636aed064823bc220d828c7ff08d1d52 +935eb8de039dec65669a96a1c3b86f4b03a1b86c +0277173b1defb63216d40a8d8805ae6d5d563c26 +2a30e67d20f76bbcd9a7d445f616f005316e0a1a +d32528e733f2711b34dbc41fbb2bb0f153bf7e9a +4cad91663df381d0dff8526f3b4aa74569dfb626 +1b06ed136f17b526360617a70026aed5ded5746c +895fbd768f0c89cea3f78acac58b233d4e3a145e +f0295becbf3ef1fb78095306408789253fe0c114 +8d573198638e52e2dbd9abc609861430f9d2bcc3 +9d9c4185fadaf243bb97c226e2fef16b65299699 +eebe4580bc8d6484d79ecb24dd87412221cf2ea7 +9cf6393a4f82b9c81d3b4b468a17a89db10531a2 +598a9c4e4dcd03c6d80fba005de729a6a3aeba7e +6970b30c6f1d2be7947295fe18f2390649b17a4b +f359afcc410432ed5d30001acda0c66741ee8935 +126000ba9e7ff16271be2f4eef3df99ade8d624f +b5e4b9b5100ec15217d43edb5f4149439f4b20a5 +b987ca4ee495a7fff82f0ac14ef0753bfb7586e2 +b03013396cb2f4bf25746388b3982a2c3616e16b +9a97f39afaa890caa7987c6bc001b9a66e3e74e8 +cad504bf4c302f7a72e0a0e191f3fdbafda7340f +45cf8a03cb57b8639a8d47323bde46ba22d9eeaf +b7450cdbd89a1c862f4d4d8bf093f8a0b5448f9c +0910cbe4ef31eb95fd76c7c2f820419fe64a3150 +92a810d04b906722c9efe60e3997243c71ff3d4c +45173fa6fca9537abb0a0554f731d14b9f89c456 +fd4ca17360e6fc0c9bb76bf6b5b07c9102c12728 +ddff3447f29b62d79a33f728791f42fa9436216e +36a5a4404836da323c755523fbd27563a8e84f94 +c991b304dee368f506cfee27ddaa333f1f82c518 +d38d1a3e75aa97ffa8755ddd431754a6d0942964 +a332a7d5a15214015f9553fdb2bcf80a1a4b8dc0 +604e08c83cf58ca7e7cda2ab284c1ace7bb12977 +18a1bbad98bd4321f15e7921d9aec91661499d90 +8049241e226c16bd07b029c0cb4b62ac40f0c923 +797441ee995aac59f55d59a93ecb55e8ecbe7dbc +62fdf9b07087b80d2142799bdd2324f61483359d +f60b4ad57912b78a96af08046a503f7905610a8c +13e31dd6548d64a5992f439e74bb424bf88aca04 +fbce66a982679b5409a295be5c99a2eef429cabf +9f2c2dba21855b8cb9b193b1819be73fa4a23a99 +a89221873a3ee2451c73b41bbe2d99d36f439d31 +3d6ad407770e13958e157bf026cae0bfb9254899 +901ba3e3819405306414628306746552b0aa1d28 +7a43fbb959c38e025e558e472ad57de357539894 +0d89fa0877930c6c8a539a656c1009ad8ab6755b +54aedc013744c86b11157423fa3cffc9a51eef02 +f0c1f8abb0182da557d07372b938f3a0a4bb906f +4ed818060ecf4a38a02c8cb48f6cbc78d2ee7708 +3bdf242fc68a8d767932c6214455d4d413effbc9 +5e468994fbb349e8eefc996954a31a67a34aaa15 +41aa9c4a801a01eca1fad22a7095372d23dace60 +2adbddb03840ad71e843c6c4a207a13e871cd1d4 +13e352dc53dec0127c5f94a60055d0ca829420dc +95e14dc81dd30ee0d396ad08dca9a6980d16eee1 +61fb80660f73e5aa5b69302ecc7ac33da206ba5a +05a761932edd05cf94ffe938908baf058f38632a +ee92243e66f2df03b3a759a8ffb75dc06f0cea0d +22cdf93c062eeaa0f8f9d6220f01b67240073dfb +76b33491596736ca804e3a29bd8398d7a1516ab7 +6e4e98ee8ce2da3cca2e2fd210e9e8dbc9b1c936 +c838283ecdfb9490425bb071b7c22e542de46c7c +5e3f5e4f25b65b583d3bfefac9e1148035781089 +f7388e93d3dd91a90239aedac4ec58404f103a2e +0a2f46b0158b6fc7244a585913b0925c0acf707f +dd561667cb7ccbbfed3134b05a565971ef6f5873 +6f01dcf63873a5e42798635ab4026c9a5f9fa213 +70fec9e36bcd1a3d93df019be084aaf89cecd7d7 +f9b74ef3fc74fd7d2aa94560820341f03cda8e12 +998c3046fab2b52bc9f141cfb588a18c05506a86 +89cc4f905e30b913ca20e4192d538cc5cbe2c38d +87d90efd69b64f769116956a5db89e536e9e3714 +5aeaa9ccd1568a77e075dbe2bd2435bd60c87c91 +bfb270acfa30713dc8c968bb9ee40cf5a2360359 +1b8c88451b0554502435d3883c528ad0aad1b09b +57ee73990f1ce29916adfd99f93eae1ccea1a43b +808c84f89d0edcef9ddaab0b849a382719f6ec9e +14b860bf64020451ced823b859da8cb912278ab9 +c63364610f4a041df1c1bd81d01b1f6856160749 +92eadc395071876d77f3babddc056b4325bdbabc +e93fff1463ae906fc986bf98c3b118c82f171546 +9ccafb1d7bdd172a9b963444072a844da379c4f7 +b4a509a3f817121c3df98ddfd96b2769e18a3e5a +dbc4ae03963014ab4b7957d62ba59dbd8f938c33 +8ddf60db7ad636b6a31b590251c671ded635fa1d +f199b8a33d9443a258a1f49a1a29674cd9ee9a20 +e542728cde676f218c552d841d0af29b92f9800b +763231051596b8e3455b839911ad6a3a1f1c3c74 +ff4cd6075b12fb32b9a906deea3ed033e3f9560a +9c3c9cdae3e20b5bdea91a0631edac5116bbc89f +93d20a734d2ee873832bed8ca5c05cf8e539c53c +ef8340d25f7c5dd5682bdecea97ce84cfce1493c +69c7ecef405d168f658a9cc7996da84c17f61e66 +4ce2f3d0d33346e9f0e96851689ee6550b2a72e3 +44e1fd926cfb0df0fbd8c41de8cd65ed8d5d6e18 +d6d2c8503c4039b682196d83a67dc28359c10c5c +ae233c4ec3d14a97c6195059f52873cdba2b4755 +0f399a9ff227896265cafab9b2e9fab6cdb9b5b9 +f4ed44ab4a8f9a87ba678d5fd1449fbf636103dc +7fcd61b2613c211bb042a82a889655178be6a212 +42973f834445d7735738bdba8847812ba3c34d95 +8df48b36ed3201d938b9974ecbee455d7dc2fb84 +96ac26e56627f0c24213fcd3a1cce9fc95f1f661 +cce94c518a46b7b0006f984bbe4d69e8749182d2 +801dd40666d1e6009920ad3ff755c7bb993b2a62 +ce829855cfca103dde55661fa1524e66b139d063 +b148803b181e30213e8a7f3bd89c8239e9dcb866 +c377feaad87f8109f85da6caf62602b30c20effc +b37cab65c63e051ebc5b491da9bd687581df94df +16e41844e7d6c5876d2caaeef6010656950c6ec5 +ee50c9e48786dea0d9df2e45805c25565c100fe3 +11dacc6154c42bc6fe3ba94c1823f8a46e4fe81a +791a0e6ddade27d1b69f4861a6640de60b9553cf +638e6c59da4fad987c437592174b188510193b2e +52f8877525d5238f3440e73710507be889d14127 +2a56baf395bf11835d784c4f8634f4525deed6a1 +bc561b4b7d6a3f71649d37d5eb9047c29efa2b13 +31809d6f8514c4a8d5677e947e3f1ebb0db210b9 +a31e9ad4f027955d43c04a05517244647e250161 +777519bd96f68c18150a0f5942f8f97a91937f5e +4eb1f39d421024d9666cec61deaf96715ffae4c6 +50fae68d416b4b8ec4ca192923dfd5ae9ea42773 +ce665863b137ac4a7470cf006a92aa7694faca71 +81f8c0378b2ab5ea0d7b65635cb529bd3c69127c +108222b9c323a05cc9339368f10ddd0859f62b43 +28f788e47e58f2b462351d6989348a4e1a241b2b +d81dccf191a48a6b59c3747d7b4ccbe3535dde40 +a90e6d2bffc422ddcdb771c53aac0bceb970a2c4 +91e49c51f1aecc9e1d75457f4920d52a4b0a133c +60dd9cc470584960431de425e2a9ffbed0e8034a +ede386c2193fc31351e193b3a8cf30030d6be62c +a084767b40c0d3ba8fa8f8d60f1e8d99a9dc3457 +3f726c99f819f97f2ab21b94d34c6b3129cd883a +77fc469fc78cdd87c29f398d46ac58dbb9ef62c0 +4ae6d0fbef60ccbecf8f23bb482e201b3678f7a3 +8858b6ddd3bce9daa08da6e05de3ca863a399c15 +22e301a3d56dc9e6878380ee92c7d19ca43119d2 +c484ec6c9b85ca4e331e395c564ae232fd0681dd +a46a671e253528e450bd57645c400bf761da07ab +655970d9c60ae6850daf452457e14e21047c0e1b +b6a48914c50631914192aa11b19205436a9c664d +7db65c363a0cc6ca7cdb04de9a973ab70013baad +6366941275344dac7e2130b0c972e90117d37ed0 +4fb2586661471a1572c2df2a5a091011d45eb7c4 +d7be7b39fa1021ec4518186afe145ee948e12a94 +85aec87b11ec41295558175c63f1f5a849460fdf +aeb31756276034dd506fdf97c8aaade0e7e584f5 +ac016e17d20253129a0287cee7e1d06b7ef15966 +bf74d377fb8e20140da6eac1407414928384bcea +2c811e08db651a4aed6ea0f7c1972d60de6de8ab +e5d26e47c7a482c072a7fe47bb84c56854734184 +96a63a3e0cefe920819bd42add0041837b1214a1 +e526ca6284b9e13be1b912b80dd73a34e739b539 +ecd21357f16106e541e9c2854ead2a906659b938 +4b5a7ce0c301ad971f383eb60f61bf9b4026efda +929fd7276c0f0c30b9416f61a6f5f35d763d81e4 +fa8a0639f7b0ce04030b72b4d5be4f0aa36fc5cb +f1f1605c22a6283bbfd757055fcf2b584a857709 +0c173a15ca1bf20999f74987988985508c9de463 +df0793f324e33066cc746c0cb1d053d35733d626 +2b0179d8a9b75397937126b36114df0dddeab40c +bf0a08be281dc42241e7f264c2a20515eb4781bb +3895e25a77363ae8b49358fb793f50fa8b271e2d +1fc783fc08bc078239537535f174ab8a489772c0 +1d4805ce04645f3203b0cfd3d66ea710e7433eb4 +d3b58704d1d325875fc605580c1c02b825c1bbcc +ed88e3194c4bc43aeafef929da7b419d03dea1ad +dd07f47b79628668e29cc0143b21e790100ee445 +65cc7aacfbfc7b747926375280a1d839e88d576b +080ec5209172ac9605f1434559dbb3c1e012b10a +416af3edf5b5ab265acf95568f2bc9eabd3d96de +e0a7801223fd573863939e76cb633f1dcc2d22c4 +4bc853b50fd9127687eb9e4f3b679dd261a4fa96 +c68a9a69278aa194fed96bd9733d32af3690a11e +c38f540298f0e188df5ed68fd56c623b9ac8331b +643fa0b22d70e459d7f7ec3d728ae4811dc5158f +e053e05c130549f43953f1d70e724dc9ce3e1b85 +75e898c094eea533d1dfaf141c6afccc3072c49f +2805d606bc46bf5589093a1b92d3542c13ce50c2 +32751807c9c06011eb689cba56b401a6302699c0 +30853e16d332816752dafcfca92147c7ffef5b54 +bea5b00cfe95cd37832305c0f93c339a22a7d79d +c871f323b418fac27bf834843ca26985010df53f +329fc1dce7a1c372c8b10c2f2f8732b2c60daff0 +1aefc94dd78d6e0c9209cb09fc16f53dedf42108 +8e5725666b519b61fcdc3141da5c6a57c1959909 +a4ca0b042365061020627a8c045cddacea3312ec +8bd16ee12fc8ef6723e0572c29b979c15b92b4f4 +87abe20fc118721cc5efdbd94a8462468cd1da2b +4b766fcdd4ca16399075d1e081a321b3b05ce516 +f6241b3e420e19f3f0507cbbc872fe9218916a02 +7ee523604851af62c0a47c07ee023a8710ef32f7 +776ba233e939fe41a74c6b2632b93a0679a32c71 +6a796b2b53fe542e0f340f250f4f20d69efed8d0 +23d78c4dd01bc74ba35db3e3df95280f6f1b2e22 +f4b15e2de97c4f8cdbb40bef4c9d0ab2807974d9 +fff72de5bf8ac7b70208e655f237b80e70e18851 +170bc2c381f86a523de2fc8b71d62ade66303c0d +314ebdfcb38d4b4c977579f787d5e1a20d068c94 +e9274839bf316b1972d80d28e45759f898edbf86 +75171f099e82e3527d7c3469b15891bd92227ec2 +3c5e6c94caf40395e031fbde44a0cca46fdd76ec +dc8fc0c73bebbc1c48ac5540026030c9cc00ec23 +492d22f92919d8d9d59568318c26c1e2ac4890cc +80c3a734298e824f9321c4efdd446086a3baad89 +47535d7c3ec79c5978cdcc03a5351ddbbb22538d +1b25b6df0f08f7474228c5b6ed13b58682e1e440 +c530c15180631cea95e9c292cf7fabde9dca9db3 +2723bcdce3248417e98e6c43207bef74d34076c1 +ed22eb4a62bd8d5369aaec87d4cbdc03c9f16368 +9111df9673beb6d6616d491a5478f09b5f14d040 +d86bb075bf6d1e78c1e4f3dd38b0ea828ef5ecfe +50a1cc0f0aef1514b917a5a3f4476967170b429d +6ce733747e160ca699711f2c47e686284ca9aa07 +b44adf92342ad4f9c343ba29c081a91687932936 +88799ea1b1c08f4bc1a487c9e3c2effd5e1650ae +080d7c700fc3291560d79fc590e05b8e2bad984f +12af74b289f8cdc6caf850dc6c802f9936b1e8b3 +8e4f7e72410df3ba430082c7cf385f26fd75b033 +8ac80412867118172dc4172494304e19969e9489 +f2734c2828f69d9cfd535e5eab0592a7674b2b61 +0b9fb682890b8fe10cec54072b809a5efe57d33d +5b029aaedb5fcf7cadd249607dd28eb3f233ab8c +79af9fbd8c3c0e54702a9c92b171f134bd4466c8 +c412fd805ddf3282dc2e1f28e30f51ffcb1f1da2 +111849345bb5140f86b48e730ceab4bff45fa2e9 +a0b1e57b20a17177ed5a9a54e4a8aab597a546b4 +ca209230c8e73745cf8cfc79f500c9c46e103306 +a230b0588788dbe1ac84622aea169c577b381241 +dfef6b6af08097f0676a2323085558fbbd3c48c6 +3192e5278abca7c1f3b4a2a7f77a0ce941c73985 +7c7ddd9ead99a8b5033a1a5d4698032c9e2b3a92 +10b930dde8f14e9cb661810e97a33bbf144fc55c +9225de2cf652fe2bf6e50636824cdb641546f57d +598ef9c44b3ea2cc142c175f077b493f39f5ba22 +c49355c7170a64bdd7864cc3ba9a64916b67fe7c +857d1e171e051b254a617f27b39f6a551054cee2 +21833f9456f6ad5bc06321ad6d9590f42ce0195c +8910b4717e5bb946ee6988f7fe9fd461f53a5935 +5703dff0939f05c7457cebd6fc61d88ab13afe41 +8bfa13b15b84cb372950fb7b25a1080173060b6a +ac23a7c1f19b3d8c326ffe75c8e13edf285f90fe +19be26afe3d04783a92d032b55bf3fb1e2ae63cc +f7ec7cfd38b543ba81ac7bed5b77f9a19739460b +36afd4db4442c45d4078b1a7ad16a1872b5bee0d +88c2ae3ed2bb5d367dd408c9255cd8f1e7a36c7d +a13a417cdcfdfd1f1b3bf997bb6ffe6e69b096b9 +d6064a89ac97dc0d2ce9da3982e1a4e25afaeda8 +7146d96de3e15a80cafbab2af48ff6f65d8e41bb +5628c70f2a44567695e5331fe2293c5b7f35b629 +7ff4a538a8682cdf02a4bcd6f15499c841001b73 +aa5fa642b0e7ce2ea55e2298886f212f11a8894e +8efd1c820b9a782d8608d54d924658536178295c +50a226563cd8d7c0a5e8448e87fede0eb72a8354 +b860915f8b0dae98e57a254d11575ea41f5c5a79 +d304fef3746039183f51b3ac8f4774dcf3a64f59 +53ab12d9318d5d195ccc77028b0e3ae66dc6e1fd +668de70be039a4f1ffcf20aeae2a22ee71fc55a8 +0fea960ca917b73aff853fe88476174c8a313863 +f89502306dcf6393a2c7b0efbb0fa728fc582137 +ff58b1c3bdff5e5f687f10f9e40ce495ca49674e +0b96abc35f1a9d46a27eeddd7df418d107c29c57 +b0b57a17306a7e963a4fe463f84e2b150a00a859 +4105cb6fd964ad13099ca83b1fdf3d35f3961f74 +23281a4dc3afc42a001346caec4dbb8193f0bb53 +8daf103fa138f9a184448ebf1c2e03b9dbd96f21 +02e5308c1b9f3771bbe49bc5036215fa2bd66aa9 +a65ced1a66575c652baf5084644b8647f531be8c +2456a835f0bc7796d9ff71f64837fa6790e2b7cc +9ec1330b455c1ab2eb6b89f8a2ab885677d4ae8a +0b738075bd43fbd4410e30a51e0498cbfd2b7513 +98c80e374b84e5a9c2d5c36889a0b1ebed5b814b +25720fc394e27a951bcad26095fb5a711bfacb8f +4cfd57d2e38207d78722ce8c9274ba8dd700d1cc +0fc1c31a878e93d938c67db3f958e82e3c39659f +df1ab5b4d67b46b5e9e840b1fbe0ff02520831f9 +5bc3b6cede8dabdf3f4f27ddb03723cbb7cde51a +c2ea1e6561caba3abffce361abc800822b9e0efe +caa2f106d704ec3ade63498031dd58d34510bc76 +dce853ef76ef90c46d84294225088d595467d08c +dbc8a8c86ae50059fddb2d6834fa5f0c9bbf9b71 +0f921e6a0492c4e9f037a9ed91f474885032d68c +041331e1da23e4136fd046ed870cdcc177464176 +e6ba5068f107ac234576e77cedbd748b665369c2 +76fcd9d5034143a5b041766552670d19f926097d +72bf1b3d0962304850a3ef5fe375db4bff1d0a39 +919db037f1f5cc73cdcaef92dd9cb0e7f5c8dec3 +c36229b0b2e9d4554053f5c9fc451ac29a493b1f +9e4bb312e6958d2baa309ba670e5eed1523c6f47 +d7ba4a233bd5a6f8fadee681c68a995e23fe36d7 +98514988a3d3e8b7dbf0463884a5c38f5ed5562d +5412c08c3cf13577566064edd04da021c37b7cbe +31bcc667863f368157efa1143a78623a5db8f0d1 +7bd1aa566fb4a4fe194f209085649f2c722b0cff +c4522e71c7e1d8ecfd70112e9375b9d00d6733a8 +e22f409f18881b63a8e747036584a71217f40e6e +97ec6e5c9098a1240655cfcab05b6cd5eedb6cd1 +bc121b0eb19713ec72002b5be03ba5ac35903a17 +c98f6b3d93a2cc1b49a6db425ea2b661089d0f9e +0de7fd36de57a68e543b4c1f184fba192c398c73 +e662d281b837c25b2b70525aa8fe8af894339823 +44adf683ad232db8ce0cb89b3e236a1f5944cfb0 +cb2ed300a89ebf9f0654da869ced665ed8b2abe7 +0a6d48d9ed60b0b02177059ab116f8f46d2cbed3 +b42291334651fff46dbfe5947a726f65cb9d7dfe +e5364991daecb73aca3bb5ac37f2619d7a89211b +4a2b170c075ce703cbdc82519a48016a9ee3f99c +924de0bd75a7f75df65d7d15f9d1587a2e794abf +1253f8692fc3a11be9430685cd405236a68df6c3 +2b799ae9e1e0a540f9a5971ddf27d83254668279 +c9bdf9a75f9fde8cd011e4aa94be4ed4347078a3 +3d69ecb4edeb80003a1a41442e320898a30dbd9c +f08222e882b18c1f279308636e03beceece2dbf1 +23e03f8d26d7bd03273a5dcbdcfe3905dfb49ffb +03dd707dc027fbf6f24120213f8eb66571600374 +d0754799698de2c032abcb8198ee5d5401063213 +072116fceb2294b97d1c40f79305f2e3ff71812b +e66cc1d58e16bf1650dd6479fed64ecaca8c6098 +f137753a2dcd8229f89d1d1ac28039364e5850b4 +61d191fbf953700ba8aeadc9c8cf4c195efbd10c +76f3c02fb01a6df98fbd8c16ac21d159d4649d37 +6013c73b3312e11b447ed387426749014716f820 +6faffb8a83db3f209a303a4464dbdd597faad5a4 +cc9e8aca5f950c78dcfeff63c441ba993c1fe12f +8ca69a2a88a77eb06149fa049ab1a7e6de38b321 +2f71490d21796594ca6f55e375558944de9db5a0 +08cc5fd666456cb476467473ed1880c90c92dedb +e31a43c725ebe641d7c219c3886eee18eebf0bb8 +52b5a8785de760a204b2b0aab19dfaf79c2c3ff0 +483e8e4f4875a1a621ec9e9df2880d3037d95ed7 +1e5799c52535a3fc20e885916f1e7ed33ecc7f46 +a82e5d8220bbc8b5d786bed99b0876f530b9b7cc +7fe6c5c993706e8395cdaf7977bee793c06f48f3 +2a0836f6d5e7c1d7e97bedb0e0ea33dcaf981f77 +ddc308068d69c6c9aa629ee3c4ce75e1d1cf08b5 +ec139a5621a9c9f03e1988391a3c7c6c5d849776 +c01a6c48b982d625fd9f4f69005878781d3d56fa +95a983d56dbda457e3bf8766d59bac74c7aa5699 +760741a00833876976389ed7a6b73f36ee5b4c13 +6e5e5abba6f8bbbe61c22795df440dfafcfdc378 +cf2cecb18779ce83de9adebf382dff1c19b12840 +af9b7a9f2f73b1a2f9728106774dd13e8d1cdd8d +115735d547fdeade822f547eb3e8c8f9961a9b07 +c2c69edf37b5c02aafa01d0407dadbf5ef8751b5 +a072d1a83787e786d074a4b5871b0b961781f7c6 +ed2cd59e258f756b2eaed7909a60956ade6ef7ee +ae5575ba41c8a782805afb1c08730343cfc22397 +6ff2c8d29f6b5a5c2ce63f0a16f3bb0dbd049451 +a80de15113166354cdf208e3d8b6e25f4511a591 +06bd4f637f15e769f088d9051a5af94bbb0217a3 +6700cc993cc07fb0f5b8b577ff8c4afcf0b18274 +37f9a1f627c0995d89b62923e75cd092600894f9 +8844ef15ded02d5ed86fb95aaf251235fcef2396 +1b87e5b5b184a0a6c683eda23b36393822b57f03 +e2bf830bb6c1bfa038c943dd6f5d92a406bd723f +423ca302a3ee87000530da3c105f269b8fabece7 +4e14afe42fdd468d5de11df8cc13defdcb8e83f8 +3e90fe6534206412ea22beaa445cf20d28fbe718 +88b77c7da0a672c89e24df37ea6e9085b4e2a05c +0ad104190465d8d65c2344bbe10dcf3df025d86c +5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a +59ad56851a342d2c62f6b38bf15002b23ab439e1 diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index e560b98d02..c60f8ab695 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -11049f4fe62606d1b0380a9ef800ac130f0fbadf +82bcf405f6db1d55b684a1f63a4aabad376cdad7 diff --git a/doc/developer-notes.md b/doc/developer-notes.md index f10ad8e877..0de1892200 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1,6 +1,43 @@ Developer Notes =============== +<!-- markdown-toc start --> +**Table of Contents** + +- [Developer Notes](#developer-notes) + - [Coding Style](#coding-style) + - [Doxygen comments](#doxygen-comments) + - [Development tips and tricks](#development-tips-and-tricks) + - [Compiling for debugging](#compiling-for-debugging) + - [Compiling for gprof profiling](#compiling-for-gprof-profiling) + - [debug.log](#debuglog) + - [Testnet and Regtest modes](#testnet-and-regtest-modes) + - [DEBUG_LOCKORDER](#debug_lockorder) + - [Valgrind suppressions file](#valgrind-suppressions-file) + - [Compiling for test coverage](#compiling-for-test-coverage) + - [Locking/mutex usage notes](#lockingmutex-usage-notes) + - [Threads](#threads) + - [Ignoring IDE/editor files](#ignoring-ideeditor-files) +- [Development guidelines](#development-guidelines) + - [General Bitcoin Core](#general-bitcoin-core) + - [Wallet](#wallet) + - [General C++](#general-c) + - [C++ data structures](#c-data-structures) + - [Strings and formatting](#strings-and-formatting) + - [Variable names](#variable-names) + - [Threads and synchronization](#threads-and-synchronization) + - [Source code organization](#source-code-organization) + - [GUI](#gui) + - [Subtrees](#subtrees) + - [Git and GitHub tips](#git-and-github-tips) + - [Scripted diffs](#scripted-diffs) + - [RPC interface guidelines](#rpc-interface-guidelines) + +<!-- markdown-toc end --> + +Coding Style +--------------- + Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to a single style, which is specified below. When writing patches, favor the new @@ -138,43 +175,44 @@ Documentation can be generated with `make docs` and cleaned up with `make clean- Development tips and tricks --------------------------- -**compiling for debugging** +### Compiling for debugging -Run configure with the --enable-debug option, then make. Or run configure with -CXXFLAGS="-g -ggdb -O0" or whatever debug flags you need. +Run configure with `--enable-debug` to add additional compiler flags that +produce better debugging builds. -**compiling for gprof profiling** +### Compiling for gprof profiling -Run configure with the --enable-gprof option, then make. +Run configure with the `--enable-gprof` option, then make. -**debug.log** +### debug.log If the code is behaving strangely, take a look in the debug.log file in the data directory; error and debugging messages are written there. -The -debug=... command-line option controls debugging; running with just -debug or -debug=1 will turn +The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn on all categories (and give you a very large debug.log file). -The Qt code routes qDebug() output to debug.log under category "qt": run with -debug=qt +The Qt code routes `qDebug()` output to debug.log under category "qt": run with `-debug=qt` to see it. -**testnet and regtest modes** +### Testnet and Regtest modes -Run with the -testnet option to run with "play bitcoins" on the test network, if you +Run with the `-testnet` option to run with "play bitcoins" on the test network, if you are testing multi-machine code that needs to operate across the internet. -If you are testing something that can run on one machine, run with the -regtest option. -In regression test mode, blocks can be created on-demand; see test/functional/ for tests -that run in -regtest mode. +If you are testing something that can run on one machine, run with the `-regtest` option. +In regression test mode, blocks can be created on-demand; see [test/functional/](/test/functional) for tests +that run in `-regtest` mode. -**DEBUG_LOCKORDER** +### DEBUG_LOCKORDER -Bitcoin Core is a multithreaded application, and deadlocks or other multithreading bugs -can be very difficult to track down. Compiling with -DDEBUG_LOCKORDER (configure -CXXFLAGS="-DDEBUG_LOCKORDER -g") inserts run-time checks to keep track of which locks -are held, and adds warnings to the debug.log file if inconsistencies are detected. +Bitcoin Core is a multi-threaded application, and deadlocks or other +multi-threading bugs can be very difficult to track down. The `--enable-debug` +configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts +run-time checks to keep track of which locks are held, and adds warnings to the +debug.log file if inconsistencies are detected. -**Valgrind suppressions file** +### Valgrind suppressions file Valgrind is a programming tool for memory debugging, memory leak detection, and profiling. The repo contains a Valgrind suppressions file @@ -189,7 +227,7 @@ $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ $ valgrind -v --leak-check=full src/bitcoind -printtoconsole ``` -**compiling for test coverage** +### Compiling for test coverage LCOV can be used to generate a test coverage report based upon `make check` execution. LCOV must be installed on your system (e.g. the `lcov` package @@ -205,22 +243,73 @@ make cov # A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. ``` +**Sanitizers** + +Bitcoin can be compiled with various "sanitizers" enabled, which add +instrumentation for issues regarding things like memory safety, thread race +conditions, or undefined behavior. This is controlled with the +`--with-sanitizers` configure flag, which should be a comma separated list of +sanitizers to enable. The sanitizer list should correspond to supported +`-fsanitize=` options in your compiler. These sanitizers have runtime overhead, +so they are most useful when testing changes or producing debugging builds. + +Some examples: + +```bash +# Enable both the address sanitizer and the undefined behavior sanitizer +./configure --with-sanitizers=address,undefined + +# Enable the thread sanitizer +./configure --with-sanitizers=thread +``` + +If you are compiling with GCC you will typically need to install corresponding +"san" libraries to actually compile with these flags, e.g. libasan for the +address sanitizer, libtsan for the thread sanitizer, and libubsan for the +undefined sanitizer. If you are missing required libraries, the configure script +will fail with a linker error when testing the sanitizer flags. + +The test suite should pass cleanly with the `thread` and `undefined` sanitizers, +but there are a number of known problems when using the `address` sanitizer. The +address sanitizer is known to fail in +[sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable +unless you also use `--disable-asm` when running configure. We would like to fix +sanitizer issues, so please send pull requests if you can fix any errors found +by the address sanitizer (or any other sanitizer). + +Not all sanitizer options can be enabled at the same time, e.g. trying to build +with `--with-sanitizers=address,thread` will fail in the configure script as +these sanitizers are mutually incompatible. Refer to your compiler manual to +learn more about these options and which sanitizers are supported by your +compiler. + +Additional resources: + + * [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) + * [LeakSanitizer](https://clang.llvm.org/docs/LeakSanitizer.html) + * [MemorySanitizer](https://clang.llvm.org/docs/MemorySanitizer.html) + * [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) + * [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) + * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) + * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki) + * [Issue #12691: Enable -fsanitize flags in Travis](https://github.com/bitcoin/bitcoin/issues/12691) + Locking/mutex usage notes ------------------------- The code is multi-threaded, and uses mutexes and the -LOCK/TRY_LOCK macros to protect data structures. +`LOCK` and `TRY_LOCK` macros to protect data structures. -Deadlocks due to inconsistent lock ordering (thread 1 locks cs_main -and then cs_wallet, while thread 2 locks them in the opposite order: -result, deadlock as each waits for the other to release its lock) are -a problem. Compile with -DDEBUG_LOCKORDER to get lock order -inconsistencies reported in the debug.log file. +Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then +`cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock +as each waits for the other to release its lock) are a problem. Compile with +`-DDEBUG_LOCKORDER` (or use `--enable-debug`) to get lock order inconsistencies +reported in the debug.log file. Re-architecting the core code so there are better-defined interfaces between the various components is a goal, with any necessary locking -done by the components (e.g. see the self-contained CBasicKeyStore class -and its cs_KeyStore lock for example). +done by the components (e.g. see the self-contained `CBasicKeyStore` class +and its `cs_KeyStore` lock for example). Threads ------- @@ -543,7 +632,10 @@ its upstream repository. Current subtrees include: - src/leveldb - - Upstream at https://github.com/google/leveldb ; Maintained by Google, but open important PRs to Core to avoid delay + - Upstream at https://github.com/google/leveldb ; Maintained by Google, but + open important PRs to Core to avoid delay. + - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when + merging upstream changes to the leveldb subtree. - src/libsecp256k1 - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintaned by Core contributors. @@ -554,6 +646,52 @@ Current subtrees include: - src/univalue - Upstream at https://github.com/jgarzik/univalue ; report important PRs to Core to avoid delay. +Upgrading LevelDB +--------------------- + +Extra care must be taken when upgrading LevelDB. This section explains issues +you must be aware of. + +### File Descriptor Counts + +In most configurations we use the default LevelDB value for `max_open_files`, +which is 1000 at the time of this writing. If LevelDB actually uses this many +file descriptors it will cause problems with Bitcoin's `select()` loop, because +it may cause new sockets to be created where the fd value is >= 1024. For this +reason, on 64-bit Unix systems we rely on an internal LevelDB optimization that +uses `mmap()` + `close()` to open table files without actually retaining +references to the table file descriptors. If you are upgrading LevelDB, you must +sanity check the changes to make sure that this assumption remains valid. + +In addition to reviewing the upstream changes in `env_posix.cc`, you can use `lsof` to +check this. For example, on Linux this command will show open `.ldb` file counts: + +```bash +$ lsof -p $(pidof bitcoind) |\ + awk 'BEGIN { fd=0; mem=0; } /ldb$/ { if ($4 == "mem") mem++; else fd++ } END { printf "mem = %s, fd = %s\n", mem, fd}' +mem = 119, fd = 0 +``` + +The `mem` value shows how many files are mmap'ed, and the `fd` value shows you +many file descriptors these files are using. You should check that `fd` is a +small number (usually 0 on 64-bit hosts). + +See the notes in the `SetMaxOpenFiles()` function in `dbwrapper.cc` for more +details. + +### Consensus Compatibility + +It is possible for LevelDB changes to inadvertently change consensus +compatibility between nodes. This happened in Bitcoin 0.8 (when LevelDB was +first introduced). When upgrading LevelDB you should review the upstream changes +to check for issues affecting consensus compatibility. + +For example, if LevelDB had a bug that accidentally prevented a key from being +returned in an edge case, and that bug was fixed upstream, the bug "fix" would +be an incompatible consensus change. In this situation the correct behavior +would be to revert the upstream fix before applying the updates to Bitcoin's +copy of LevelDB. In general you should be wary of any upstream changes affecting +what data is returned from LevelDB queries. Git and GitHub tips --------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 7c2fe56d9d..72e5cdb95d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,8 +4,8 @@ DIST_SUBDIRS = secp256k1 univalue -AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) -AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) +AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) +AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_CPPFLAGS = $(HARDENED_CPPFLAGS) EXTRA_LIBRARIES = @@ -162,6 +162,7 @@ BITCOIN_CORE_H = \ validation.h \ validationinterface.h \ versionbits.h \ + walletinitinterface.h \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 675f3c28af..e4620e63c6 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -22,8 +22,8 @@ bool SerializeDB(Stream& stream, const Data& data) // Write and commit header, data try { CHashWriter hasher(SER_DISK, CLIENT_VERSION); - stream << FLATDATA(Params().MessageStart()) << data; - hasher << FLATDATA(Params().MessageStart()) << data; + stream << Params().MessageStart() << data; + hasher << Params().MessageStart() << data; stream << hasher.GetHash(); } catch (const std::exception& e) { return error("%s: Serialize or I/O error - %s", __func__, e.what()); @@ -66,7 +66,7 @@ bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) CHashVerifier<Stream> verifier(&stream); // de-serialize file header (network specific magic number) and .. unsigned char pchMsgTmp[4]; - verifier >> FLATDATA(pchMsgTmp); + verifier >> pchMsgTmp; // ... verify the network matches ours if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) return error("%s: Invalid network magic number", __func__); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index db5e610684..df7fad89c2 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -18,6 +18,10 @@ #include <httpserver.h> #include <httprpc.h> #include <utilstrencodings.h> +#if ENABLE_WALLET +#include <wallet/init.h> +#endif +#include <walletinitinterface.h> #include <boost/thread.hpp> @@ -59,6 +63,12 @@ bool AppInit(int argc, char* argv[]) { bool fRet = false; +#if ENABLE_WALLET + g_wallet_init_interface.reset(new WalletInit); +#else + g_wallet_init_interface.reset(new DummyWalletInit); +#endif + // // Parameters // diff --git a/src/compressor.cpp b/src/compressor.cpp index 86de2900e9..da639a01af 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -9,7 +9,15 @@ #include <pubkey.h> #include <script/standard.h> -bool CScriptCompressor::IsToKeyID(CKeyID &hash) const +/* + * These check for scripts for which a special case with a shorter encoding is defined. + * They are implemented separately from the CScript test, as these test for exact byte + * sequence correspondences, and are more strict. For example, IsToPubKey also verifies + * whether the public key is valid (as invalid ones cannot be represented in compressed + * form). + */ + +static bool IsToKeyID(const CScript& script, CKeyID &hash) { if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160 && script[2] == 20 && script[23] == OP_EQUALVERIFY @@ -20,7 +28,7 @@ bool CScriptCompressor::IsToKeyID(CKeyID &hash) const return false; } -bool CScriptCompressor::IsToScriptID(CScriptID &hash) const +static bool IsToScriptID(const CScript& script, CScriptID &hash) { if (script.size() == 23 && script[0] == OP_HASH160 && script[1] == 20 && script[22] == OP_EQUAL) { @@ -30,7 +38,7 @@ bool CScriptCompressor::IsToScriptID(CScriptID &hash) const return false; } -bool CScriptCompressor::IsToPubKey(CPubKey &pubkey) const +static bool IsToPubKey(const CScript& script, CPubKey &pubkey) { if (script.size() == 35 && script[0] == 33 && script[34] == OP_CHECKSIG && (script[1] == 0x02 || script[1] == 0x03)) { @@ -45,24 +53,24 @@ bool CScriptCompressor::IsToPubKey(CPubKey &pubkey) const return false; } -bool CScriptCompressor::Compress(std::vector<unsigned char> &out) const +bool CompressScript(const CScript& script, std::vector<unsigned char> &out) { CKeyID keyID; - if (IsToKeyID(keyID)) { + if (IsToKeyID(script, keyID)) { out.resize(21); out[0] = 0x00; memcpy(&out[1], &keyID, 20); return true; } CScriptID scriptID; - if (IsToScriptID(scriptID)) { + if (IsToScriptID(script, scriptID)) { out.resize(21); out[0] = 0x01; memcpy(&out[1], &scriptID, 20); return true; } CPubKey pubkey; - if (IsToPubKey(pubkey)) { + if (IsToPubKey(script, pubkey)) { out.resize(33); memcpy(&out[1], &pubkey[1], 32); if (pubkey[0] == 0x02 || pubkey[0] == 0x03) { @@ -76,7 +84,7 @@ bool CScriptCompressor::Compress(std::vector<unsigned char> &out) const return false; } -unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const +unsigned int GetSpecialScriptSize(unsigned int nSize) { if (nSize == 0 || nSize == 1) return 20; @@ -85,7 +93,7 @@ unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const return 0; } -bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigned char> &in) +bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &in) { switch(nSize) { case 0x00: @@ -139,7 +147,7 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigne // * if e==9, we only know the resulting number is not zero, so output 1 + 10*(n - 1) + 9 // (this is decodable, as d is in [1-9] and e is in [0-9]) -uint64_t CTxOutCompressor::CompressAmount(uint64_t n) +uint64_t CompressAmount(uint64_t n) { if (n == 0) return 0; @@ -158,7 +166,7 @@ uint64_t CTxOutCompressor::CompressAmount(uint64_t n) } } -uint64_t CTxOutCompressor::DecompressAmount(uint64_t x) +uint64_t DecompressAmount(uint64_t x) { // x = 0 OR x = 1+10*(9*n + d - 1) + e OR x = 1+10*(n - 1) + 9 if (x == 0) diff --git a/src/compressor.h b/src/compressor.h index 6fcecd27e9..561c8e66d0 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -14,6 +14,13 @@ class CKeyID; class CPubKey; class CScriptID; +bool CompressScript(const CScript& script, std::vector<unsigned char> &out); +unsigned int GetSpecialScriptSize(unsigned int nSize); +bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); + +uint64_t CompressAmount(uint64_t nAmount); +uint64_t DecompressAmount(uint64_t nAmount); + /** Compact serializer for scripts. * * It detects common cases and encodes them much more efficiently. @@ -37,28 +44,13 @@ private: static const unsigned int nSpecialScripts = 6; CScript &script; -protected: - /** - * These check for scripts for which a special case with a shorter encoding is defined. - * They are implemented separately from the CScript test, as these test for exact byte - * sequence correspondences, and are more strict. For example, IsToPubKey also verifies - * whether the public key is valid (as invalid ones cannot be represented in compressed - * form). - */ - bool IsToKeyID(CKeyID &hash) const; - bool IsToScriptID(CScriptID &hash) const; - bool IsToPubKey(CPubKey &pubkey) const; - - bool Compress(std::vector<unsigned char> &out) const; - unsigned int GetSpecialSize(unsigned int nSize) const; - bool Decompress(unsigned int nSize, const std::vector<unsigned char> &out); public: explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { } template<typename Stream> void Serialize(Stream &s) const { std::vector<unsigned char> compr; - if (Compress(compr)) { + if (CompressScript(script, compr)) { s << CFlatData(compr); return; } @@ -72,9 +64,9 @@ public: unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { - std::vector<unsigned char> vch(GetSpecialSize(nSize), 0x00); + std::vector<unsigned char> vch(GetSpecialScriptSize(nSize), 0x00); s >> CFlatData(vch); - Decompress(nSize, vch); + DecompressScript(script, nSize, vch); return; } nSize -= nSpecialScripts; @@ -96,9 +88,6 @@ private: CTxOut &txout; public: - static uint64_t CompressAmount(uint64_t nAmount); - static uint64_t DecompressAmount(uint64_t nAmount); - explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } ADD_SERIALIZE_METHODS; diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index fb0d4215a2..752f985bc0 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -71,6 +71,31 @@ public: } }; +static void SetMaxOpenFiles(leveldb::Options *options) { + // On most platforms the default setting of max_open_files (which is 1000) + // is optimal. On Windows using a large file count is OK because the handles + // do not interfere with select() loops. On 64-bit Unix hosts this value is + // also OK, because up to that amount LevelDB will use an mmap + // implementation that does not use extra file descriptors (the fds are + // closed after being mmaped). + // + // Increasing the value beyond the default is dangerous because LevelDB will + // fall back to a non-mmap implementation when the file count is too large. + // On 32-bit Unix host we should decrease the value because the handles use + // up real fds, and we want to avoid fd exhaustion issues. + // + // See PR #12495 for further discussion. + + int default_open_files = options->max_open_files; +#ifndef WIN32 + if (sizeof(void*) < 8) { + options->max_open_files = 64; + } +#endif + LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n", + options->max_open_files, default_open_files); +} + static leveldb::Options GetOptions(size_t nCacheSize) { leveldb::Options options; @@ -78,13 +103,13 @@ static leveldb::Options GetOptions(size_t nCacheSize) options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously options.filter_policy = leveldb::NewBloomFilterPolicy(10); options.compression = leveldb::kNoCompression; - options.max_open_files = 64; options.info_log = new CBitcoinLevelDBLogger(); if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. options.paranoid_checks = true; } + SetMaxOpenFiles(&options); return options; } @@ -159,12 +184,12 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) const bool log_memory = LogAcceptCategory(BCLog::LEVELDB); double mem_before = 0; if (log_memory) { - mem_before = DynamicMemoryUsage() / 1024 / 1024; + mem_before = DynamicMemoryUsage() / 1024.0 / 1024; } leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); dbwrapper_private::HandleError(status); if (log_memory) { - double mem_after = DynamicMemoryUsage() / 1024 / 1024; + double mem_after = DynamicMemoryUsage() / 1024.0 / 1024; LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n", m_name, mem_before, mem_after); } diff --git a/src/init.cpp b/src/init.cpp index b0ba0cb835..3eb2a83243 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -43,10 +43,8 @@ #include <util.h> #include <utilmoneystr.h> #include <validationinterface.h> -#ifdef ENABLE_WALLET -#include <wallet/init.h> -#endif #include <warnings.h> +#include <walletinitinterface.h> #include <stdint.h> #include <stdio.h> #include <memory> @@ -74,6 +72,7 @@ static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; std::unique_ptr<CConnman> g_connman; std::unique_ptr<PeerLogicValidation> peerLogic; +std::unique_ptr<WalletInitInterface> g_wallet_init_interface; #if ENABLE_ZMQ static CZMQNotificationInterface* pzmqNotificationInterface = nullptr; @@ -116,7 +115,6 @@ static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; // std::atomic<bool> fRequestShutdown(false); -std::atomic<bool> fDumpMempoolLater(false); void StartShutdown() { @@ -189,9 +187,7 @@ void Shutdown() StopREST(); StopRPC(); StopHTTPServer(); -#ifdef ENABLE_WALLET - FlushWallets(); -#endif + g_wallet_init_interface->Flush(); StopMapPort(); // Because these depend on each-other, we make sure that neither can be @@ -208,7 +204,7 @@ void Shutdown() threadGroup.interrupt_all(); threadGroup.join_all(); - if (fDumpMempoolLater && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); } @@ -249,9 +245,7 @@ void Shutdown() pcoinsdbview.reset(); pblocktree.reset(); } -#ifdef ENABLE_WALLET - StopWallets(); -#endif + g_wallet_init_interface->Stop(); #if ENABLE_ZMQ if (pzmqNotificationInterface) { @@ -271,9 +265,8 @@ void Shutdown() UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); GetMainSignals().UnregisterWithMempoolSignals(mempool); -#ifdef ENABLE_WALLET - CloseWallets(); -#endif + g_wallet_init_interface->Close(); + g_wallet_init_interface.reset(); globalVerifyHandle.reset(); ECC_Stop(); LogPrintf("%s: done\n", __func__); @@ -416,9 +409,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-whitelist=<IP address or network>", _("Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway")); -#ifdef ENABLE_WALLET - strUsage += GetWalletHelpString(showDebug); -#endif + strUsage += g_wallet_init_interface->GetHelpString(showDebug); #if ENABLE_ZMQ strUsage += HelpMessageGroup(_("ZeroMQ notification options:")); @@ -692,8 +683,8 @@ void ThreadImport(std::vector<fs::path> vImportFiles) } // End scope of CImportingNow if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { LoadMempool(); - fDumpMempoolLater = !fRequestShutdown; } + g_is_mempool_loaded = !fRequestShutdown; } /** Sanity checks @@ -1087,10 +1078,7 @@ bool AppInitParameterInteraction() return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); -#ifdef ENABLE_WALLET - if (!WalletParameterInteraction()) - return false; -#endif + if (!g_wallet_init_interface->ParameterInteraction()) return false; fIsBareMultisigStd = gArgs.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); @@ -1256,9 +1244,7 @@ bool AppInitMain() * available in the GUI RPC console even if external calls are disabled. */ RegisterAllCoreRPCCommands(tableRPC); -#ifdef ENABLE_WALLET - RegisterWalletRPC(tableRPC); -#endif + g_wallet_init_interface->RegisterRPC(tableRPC); /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections @@ -1275,10 +1261,8 @@ bool AppInitMain() int64_t nStart; // ********************************************************* Step 5: verify wallet database integrity -#ifdef ENABLE_WALLET - if (!VerifyWallets()) - return false; -#endif + if (!g_wallet_init_interface->Verify()) return false; + // ********************************************************* Step 6: network initialization // Note that we absolutely cannot open any actual connections // until the very end ("start node") as the UTXO/block state @@ -1596,12 +1580,7 @@ bool AppInitMain() fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet -#ifdef ENABLE_WALLET - if (!OpenWallets()) - return false; -#else - LogPrintf("No wallet support compiled in!\n"); -#endif + if (!g_wallet_init_interface->Open()) return false; // ********************************************************* Step 9: data directory maintenance @@ -1747,9 +1726,7 @@ bool AppInitMain() SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading")); -#ifdef ENABLE_WALLET - StartWallets(scheduler); -#endif + g_wallet_init_interface->Start(scheduler); return true; } diff --git a/src/init.h b/src/init.h index 6f75a43e62..c93a210154 100644 --- a/src/init.h +++ b/src/init.h @@ -6,11 +6,15 @@ #ifndef BITCOIN_INIT_H #define BITCOIN_INIT_H +#include <memory> #include <string> class CScheduler; class CWallet; +class WalletInitInterface; +extern std::unique_ptr<WalletInitInterface> g_wallet_init_interface; + namespace boost { class thread_group; diff --git a/src/netaddress.h b/src/netaddress.h index 93bbb66491..ad6b55eb58 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -93,7 +93,7 @@ class CNetAddr template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(FLATDATA(ip)); + READWRITE(ip); } friend class CSubNet; @@ -131,8 +131,8 @@ class CSubNet template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(network); - READWRITE(FLATDATA(netmask)); - READWRITE(FLATDATA(valid)); + READWRITE(netmask); + READWRITE(valid); } }; @@ -166,7 +166,7 @@ class CService : public CNetAddr template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(FLATDATA(ip)); + READWRITE(ip); unsigned short portN = htons(port); READWRITE(FLATDATA(portN)); if (ser_action.ForRead()) diff --git a/src/protocol.h b/src/protocol.h index e518d11944..a07c5ea862 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -48,10 +48,10 @@ public: template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(FLATDATA(pchMessageStart)); - READWRITE(FLATDATA(pchCommand)); + READWRITE(pchMessageStart); + READWRITE(pchCommand); READWRITE(nMessageSize); - READWRITE(FLATDATA(pchChecksum)); + READWRITE(pchChecksum); } char pchMessageStart[MESSAGE_START_SIZE]; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index f1298df8ca..f853c04617 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -34,8 +34,10 @@ #include <warnings.h> #ifdef ENABLE_WALLET +#include <wallet/init.h> #include <wallet/wallet.h> #endif +#include <walletinitinterface.h> #include <stdint.h> @@ -677,6 +679,11 @@ int main(int argc, char *argv[]) // Start up the payment server early, too, so impatient users that click on // bitcoin: links repeatedly have their payment requests routed to this process: app.createPaymentServer(); + + // Hook up the wallet init interface + g_wallet_init_interface.reset(new WalletInit); +#else + g_wallet_init_interface.reset(new DummyWalletInit); #endif /// 9. Main GUI initialization diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e15dad2f0b..31cbec4c86 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1607,13 +1607,17 @@ UniValue savemempool(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( "savemempool\n" - "\nDumps the mempool to disk.\n" + "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n" "\nExamples:\n" + HelpExampleCli("savemempool", "") + HelpExampleRpc("savemempool", "") ); } + if (!g_is_mempool_loaded) { + throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); + } + if (!DumpMempool()) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 20bfd3f355..77040f75fd 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1023,18 +1023,18 @@ UniValue signrawtransaction(const JSONRPCRequest& request) new_request.params.push_back(request.params[1]); new_request.params.push_back(request.params[3]); return signrawtransactionwithkey(new_request); - } - // Otherwise sign with the wallet which does not take a privkeys parameter + } else { #ifdef ENABLE_WALLET - else { + // Otherwise sign with the wallet which does not take a privkeys parameter new_request.params.push_back(request.params[0]); new_request.params.push_back(request.params[1]); new_request.params.push_back(request.params[3]); return signrawtransactionwithwallet(new_request); - } +#else + // If we have made it this far, then wallet is disabled and no private keys were given, so fail here. + throw JSONRPCError(RPC_INVALID_PARAMETER, "No private keys available."); #endif - // If we have made it this far, then wallet is disabled and no private keys were given, so fail here. - throw JSONRPCError(RPC_INVALID_PARAMETER, "No private keys available."); + } } UniValue sendrawtransaction(const JSONRPCRequest& request) diff --git a/src/serialize.h b/src/serialize.h index 91da6b0f80..247e915298 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -59,6 +59,12 @@ inline T* NCONST_PTR(const T* val) return const_cast<T*>(val); } +//! Safely convert odd char pointer types to standard ones. +inline char* CharCast(char* c) { return c; } +inline char* CharCast(unsigned char* c) { return (char*)c; } +inline const char* CharCast(const char* c) { return c; } +inline const char* CharCast(const unsigned char* c) { return (const char*)c; } + /* * Lowest-level serialization and conversion. * @note Sizes of these types are verified in the tests @@ -177,6 +183,8 @@ template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_wri template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); } template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } +template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); } +template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); } template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); } @@ -189,6 +197,8 @@ template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); } template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } +template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); } +template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); } template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); } template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; } diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 3c26013622..127cc154df 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -25,16 +25,16 @@ BOOST_FIXTURE_TEST_SUITE(compress_tests, BasicTestingSetup) bool static TestEncode(uint64_t in) { - return in == CTxOutCompressor::DecompressAmount(CTxOutCompressor::CompressAmount(in)); + return in == DecompressAmount(CompressAmount(in)); } bool static TestDecode(uint64_t in) { - return in == CTxOutCompressor::CompressAmount(CTxOutCompressor::DecompressAmount(in)); + return in == CompressAmount(DecompressAmount(in)); } bool static TestPair(uint64_t dec, uint64_t enc) { - return CTxOutCompressor::CompressAmount(dec) == enc && - CTxOutCompressor::DecompressAmount(enc) == dec; + return CompressAmount(dec) == enc && + DecompressAmount(enc) == dec; } BOOST_AUTO_TEST_CASE(compress_amounts) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index e03234060d..6552613c04 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -64,7 +64,7 @@ public: CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) { CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); - ssPeersIn << FLATDATA(Params().MessageStart()); + ssPeersIn << Params().MessageStart(); ssPeersIn << _addrman; std::string str = ssPeersIn.str(); std::vector<unsigned char> vchData(str.begin(), str.end()); @@ -110,7 +110,7 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; - ssPeers1 >> FLATDATA(pchMsgTmp); + ssPeers1 >> pchMsgTmp; ssPeers1 >> addrman1; } catch (const std::exception& e) { exceptionThrown = true; @@ -142,7 +142,7 @@ BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; - ssPeers1 >> FLATDATA(pchMsgTmp); + ssPeers1 >> pchMsgTmp; ssPeers1 >> addrman1; } catch (const std::exception& e) { exceptionThrown = true; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 7a79a77e8b..9b8b7bdc56 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -19,11 +19,15 @@ protected: int intval; bool boolval; std::string stringval; - const char* charstrval; + char charstrval[16]; CTransactionRef txval; public: CSerializeMethodsTestSingle() = default; - CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), charstrval(charstrvalin), txval(MakeTransactionRef(txvalin)){} + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, CTransaction txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), txval(MakeTransactionRef(txvalin)) + { + memcpy(charstrval, charstrvalin, sizeof(charstrval)); + } + ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> @@ -31,7 +35,7 @@ public: READWRITE(intval); READWRITE(boolval); READWRITE(stringval); - READWRITE(FLATDATA(charstrval)); + READWRITE(charstrval); READWRITE(txval); } @@ -53,7 +57,7 @@ public: template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval, boolval, stringval, FLATDATA(charstrval), txval); + READWRITE(intval, boolval, stringval, charstrval, txval); } }; @@ -344,7 +348,7 @@ BOOST_AUTO_TEST_CASE(class_methods) int intval(100); bool boolval(true); std::string stringval("testing"); - const char* charstrval("testing charstr"); + const char charstrval[16] = "testing charstr"; CMutableTransaction txval; CSerializeMethodsTestSingle methodtest1(intval, boolval, stringval, charstrval, txval); CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, txval); @@ -360,7 +364,7 @@ BOOST_AUTO_TEST_CASE(class_methods) BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); - CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, FLATDATA(charstrval), txval); + CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, charstrval, txval); ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 1108dab584..5d057108b1 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -57,16 +57,16 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer) BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); vch.clear(); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, FLATDATA(bytes)); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, FLATDATA(bytes)); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); vch.clear(); vch.resize(4, 8); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, FLATDATA(bytes), b); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, FLATDATA(bytes), b); + CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); vch.clear(); } diff --git a/src/validation.cpp b/src/validation.cpp index 839b248143..77764ff923 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -227,6 +227,7 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); +std::atomic_bool g_is_mempool_loaded{false}; /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; @@ -1077,7 +1078,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes // Write index header unsigned int nSize = GetSerializeSize(fileout, block); - fileout << FLATDATA(messageStart) << nSize; + fileout << messageStart << nSize; // Write block long fileOutPos = ftell(fileout.Get()); @@ -1441,7 +1442,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint // Write index header unsigned int nSize = GetSerializeSize(fileout, blockundo); - fileout << FLATDATA(messageStart) << nSize; + fileout << messageStart << nSize; // Write undo data long fileOutPos = ftell(fileout.Get()); @@ -4283,7 +4284,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; blkdat.FindByte(chainparams.MessageStart()[0]); nRewind = blkdat.GetPos()+1; - blkdat >> FLATDATA(buf); + blkdat >> buf; if (memcmp(buf, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) continue; // read size diff --git a/src/validation.h b/src/validation.h index 0a78075316..4031989f00 100644 --- a/src/validation.h +++ b/src/validation.h @@ -158,6 +158,7 @@ extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; +extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap& mapBlockIndex; extern uint64_t nLastBlockTx; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 61481e01b6..3d7bb674f0 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -14,7 +14,7 @@ #include <wallet/wallet.h> #include <wallet/walletutil.h> -std::string GetWalletHelpString(bool showDebug) +std::string WalletInit::GetHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); strUsage += HelpMessageOpt("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE))); @@ -56,7 +56,7 @@ std::string GetWalletHelpString(bool showDebug) return strUsage; } -bool WalletParameterInteraction() +bool WalletInit::ParameterInteraction() { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { for (const std::string& wallet : gArgs.GetArgs("-wallet")) { @@ -184,7 +184,7 @@ bool WalletParameterInteraction() return true; } -void RegisterWalletRPC(CRPCTable &t) +void WalletInit::RegisterRPC(CRPCTable &t) { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { return; @@ -193,7 +193,7 @@ void RegisterWalletRPC(CRPCTable &t) RegisterWalletRPCCommands(t); } -bool VerifyWallets() +bool WalletInit::Verify() { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { return true; @@ -268,7 +268,7 @@ bool VerifyWallets() return true; } -bool OpenWallets() +bool WalletInit::Open() { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { LogPrintf("Wallet disabled!\n"); @@ -286,25 +286,29 @@ bool OpenWallets() return true; } -void StartWallets(CScheduler& scheduler) { +void WalletInit::Start(CScheduler& scheduler) +{ for (CWalletRef pwallet : vpwallets) { pwallet->postInitProcess(scheduler); } } -void FlushWallets() { +void WalletInit::Flush() +{ for (CWalletRef pwallet : vpwallets) { pwallet->Flush(false); } } -void StopWallets() { +void WalletInit::Stop() +{ for (CWalletRef pwallet : vpwallets) { pwallet->Flush(true); } } -void CloseWallets() { +void WalletInit::Close() +{ for (CWalletRef pwallet : vpwallets) { delete pwallet; } diff --git a/src/wallet/init.h b/src/wallet/init.h index 0b3ee2dda2..f8be90d3e3 100644 --- a/src/wallet/init.h +++ b/src/wallet/init.h @@ -6,38 +6,43 @@ #ifndef BITCOIN_WALLET_INIT_H #define BITCOIN_WALLET_INIT_H +#include <walletinitinterface.h> #include <string> class CRPCTable; class CScheduler; -//! Return the wallets help message. -std::string GetWalletHelpString(bool showDebug); +class WalletInit : public WalletInitInterface { +public: -//! Wallets parameter interaction -bool WalletParameterInteraction(); + //! Return the wallets help message. + std::string GetHelpString(bool showDebug) override; -//! Register wallet RPCs. -void RegisterWalletRPC(CRPCTable &tableRPC); + //! Wallets parameter interaction + bool ParameterInteraction() override; -//! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -// This function will perform salvage on the wallet if requested, as long as only one wallet is -// being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). -bool VerifyWallets(); + //! Register wallet RPCs. + void RegisterRPC(CRPCTable &tableRPC) override; -//! Load wallet databases. -bool OpenWallets(); + //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. + // This function will perform salvage on the wallet if requested, as long as only one wallet is + // being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). + bool Verify() override; -//! Complete startup of wallets. -void StartWallets(CScheduler& scheduler); + //! Load wallet databases. + bool Open() override; -//! Flush all wallets in preparation for shutdown. -void FlushWallets(); + //! Complete startup of wallets. + void Start(CScheduler& scheduler) override; -//! Stop all wallets. Wallets will be flushed first. -void StopWallets(); + //! Flush all wallets in preparation for shutdown. + void Flush() override; -//! Close all wallets. -void CloseWallets(); + //! Stop all wallets. Wallets will be flushed first. + void Stop() override; + + //! Close all wallets. + void Close() override; +}; #endif // BITCOIN_WALLET_INIT_H diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h new file mode 100644 index 0000000000..47e4e2cce1 --- /dev/null +++ b/src/walletinitinterface.h @@ -0,0 +1,51 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef WALLETINITINTERFACE_H +#define WALLETINITINTERFACE_H + +#include <string> + +class CScheduler; +class CRPCTable; + +class WalletInitInterface { +public: + /** Get wallet help string */ + virtual std::string GetHelpString(bool showDebug) = 0; + /** Check wallet parameter interaction */ + virtual bool ParameterInteraction() = 0; + /** Register wallet RPC*/ + virtual void RegisterRPC(CRPCTable &) = 0; + /** Verify wallets */ + virtual bool Verify() = 0; + /** Open wallets*/ + virtual bool Open() = 0; + /** Start wallets*/ + virtual void Start(CScheduler& scheduler) = 0; + /** Flush Wallets*/ + virtual void Flush() = 0; + /** Stop Wallets*/ + virtual void Stop() = 0; + /** Close wallets */ + virtual void Close() = 0; + + virtual ~WalletInitInterface() {} +}; + +class DummyWalletInit : public WalletInitInterface { +public: + + std::string GetHelpString(bool showDebug) override {return std::string{};} + bool ParameterInteraction() override {return true;} + void RegisterRPC(CRPCTable &) override {} + bool Verify() override {return true;} + bool Open() override {return true;} + void Start(CScheduler& scheduler) override {} + void Flush() override {} + void Stop() override {} + void Close() override {} +}; + +#endif // WALLETINITINTERFACE_H diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index fe9bbda14b..181c7f3369 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -2,36 +2,59 @@ # Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test block processing. - -This reimplements tests from the bitcoinj/FullBlockTestGenerator used -by the pull-tester. - -We use the testing framework in which we expect a particular answer from -each test. -""" - -from test_framework.test_framework import ComparisonTestFramework -from test_framework.util import * -from test_framework.comptool import TestManager, TestInstance, RejectResult -from test_framework.blocktools import * +"""Test block processing.""" +import copy +import struct import time + +from test_framework.blocktools import create_block, create_coinbase, create_transaction, get_legacy_sigopcount_block from test_framework.key import CECKey -from test_framework.script import * -from test_framework.mininode import network_thread_start -import struct +from test_framework.messages import ( + CBlock, + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxOut, + MAX_BLOCK_BASE_SIZE, + uint256_from_compact, + uint256_from_str, +) +from test_framework.mininode import P2PDataStore, network_thread_start, network_thread_join +from test_framework.script import ( + CScript, + MAX_SCRIPT_ELEMENT_SIZE, + OP_2DUP, + OP_CHECKMULTISIG, + OP_CHECKMULTISIGVERIFY, + OP_CHECKSIG, + OP_CHECKSIGVERIFY, + OP_ELSE, + OP_ENDIF, + OP_EQUAL, + OP_FALSE, + OP_HASH160, + OP_IF, + OP_INVALIDOPCODE, + OP_RETURN, + OP_TRUE, + SIGHASH_ALL, + SignatureHash, + hash160, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +MAX_BLOCK_SIGOPS = 20000 class PreviousSpendableOutput(): - def __init__(self, tx = CTransaction(), n = -1): + def __init__(self, tx=CTransaction(), n=-1): self.tx = tx self.n = n # the output we're spending # Use this class for tests that require behavior other than normal "mininode" behavior. # For now, it is used to serialize a bloated varint (b64). class CBrokenBlock(CBlock): - def __init__(self, header=None): - super(CBrokenBlock, self).__init__(header) - def initialize(self, base_block): self.vtx = copy.deepcopy(base_block.vtx) self.hashMerkleRoot = self.calc_merkle_root() @@ -48,381 +71,272 @@ class CBrokenBlock(CBlock): return r def normal_serialize(self): - r = b"" - r += super(CBrokenBlock, self).serialize() - return r + return super().serialize() -class FullBlockTest(ComparisonTestFramework): - # Can either run this test as 1 node with expected answers, or two and compare them. - # Change the "outcome" variable from each TestInstance object to only do the comparison. +class FullBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + self.extra_args = [[]] + + def run_test(self): + node = self.nodes[0] # convenience reference to the node + + # reconnect_p2p() expects the network thread to be running + network_thread_start() + + self.reconnect_p2p() + self.block_heights = {} self.coinbase_key = CECKey() self.coinbase_key.set_secretbytes(b"horsebattery") self.coinbase_pubkey = self.coinbase_key.get_pubkey() self.tip = None self.blocks = {} - - def add_options(self, parser): - super().add_options(parser) - parser.add_option("--runbarelyexpensive", dest="runbarelyexpensive", default=True) - - def run_test(self): - self.test = TestManager(self, self.options.tmpdir) - self.test.add_all_connections(self.nodes) - network_thread_start() - self.test.run() - - def add_transactions_to_block(self, block, tx_list): - [ tx.rehash() for tx in tx_list ] - block.vtx.extend(tx_list) - - # this is a little handier to use than the version in blocktools.py - def create_tx(self, spend_tx, n, value, script=CScript([OP_TRUE])): - tx = create_transaction(spend_tx, n, b"", value, script) - return tx - - # sign a transaction, using the key we know about - # this signs input 0 in tx, which is assumed to be spending output n in spend_tx - def sign_tx(self, tx, spend_tx, n): - scriptPubKey = bytearray(spend_tx.vout[n].scriptPubKey) - if (scriptPubKey[0] == OP_TRUE): # an anyone-can-spend - tx.vin[0].scriptSig = CScript() - return - (sighash, err) = SignatureHash(spend_tx.vout[n].scriptPubKey, tx, 0, SIGHASH_ALL) - tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))]) - - def create_and_sign_transaction(self, spend_tx, n, value, script=CScript([OP_TRUE])): - tx = self.create_tx(spend_tx, n, value, script) - self.sign_tx(tx, spend_tx, n) - tx.rehash() - return tx - - def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True): - if self.tip == None: - base_block_hash = self.genesis_hash - block_time = int(time.time())+1 - else: - base_block_hash = self.tip.sha256 - block_time = self.tip.nTime + 1 - # First create the coinbase - height = self.block_heights[base_block_hash] + 1 - coinbase = create_coinbase(height, self.coinbase_pubkey) - coinbase.vout[0].nValue += additional_coinbase_value - coinbase.rehash() - if spend == None: - block = create_block(base_block_hash, coinbase, block_time) - else: - coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 # all but one satoshi to fees - coinbase.rehash() - block = create_block(base_block_hash, coinbase, block_time) - tx = create_transaction(spend.tx, spend.n, b"", 1, script) # spend 1 satoshi - self.sign_tx(tx, spend.tx, spend.n) - self.add_transactions_to_block(block, [tx]) - block.hashMerkleRoot = block.calc_merkle_root() - if solve: - block.solve() - self.tip = block - self.block_heights[block.sha256] = height - assert number not in self.blocks - self.blocks[number] = block - return block - - def get_tests(self): self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) self.block_heights[self.genesis_hash] = 0 - spendable_outputs = [] - - # save the current tip so it can be spent by a later block - def save_spendable_output(): - spendable_outputs.append(self.tip) - - # get an output that we previously marked as spendable - def get_spendable_output(): - return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0) - - # returns a test case that asserts that the current tip was accepted - def accepted(): - return TestInstance([[self.tip, True]]) - - # returns a test case that asserts that the current tip was rejected - def rejected(reject = None): - if reject is None: - return TestInstance([[self.tip, False]]) - else: - return TestInstance([[self.tip, reject]]) - - # move the tip back to a previous block - def tip(number): - self.tip = self.blocks[number] - - # adds transactions to the block and updates state - def update_block(block_number, new_transactions): - block = self.blocks[block_number] - self.add_transactions_to_block(block, new_transactions) - old_sha256 = block.sha256 - block.hashMerkleRoot = block.calc_merkle_root() - block.solve() - # Update the internal state just like in next_block - self.tip = block - if block.sha256 != old_sha256: - self.block_heights[block.sha256] = self.block_heights[old_sha256] - del self.block_heights[old_sha256] - self.blocks[block_number] = block - return block - - # shorthand for functions - block = self.next_block - create_tx = self.create_tx - create_and_sign_tx = self.create_and_sign_transaction - - # these must be updated if consensus changes - MAX_BLOCK_SIGOPS = 20000 - + self.spendable_outputs = [] # Create a new block - block(0) - save_spendable_output() - yield accepted() - + b0 = self.next_block(0) + self.save_spendable_output() + self.sync_blocks([b0]) - # Now we need that block to mature so we can spend the coinbase. - test = TestInstance(sync_every_block=False) + # Allow the block to mature + blocks = [] for i in range(99): - block(5000 + i) - test.blocks_and_transactions.append([self.tip, True]) - save_spendable_output() - yield test + blocks.append(self.next_block(5000 + i)) + self.save_spendable_output() + self.sync_blocks(blocks) # collect spendable outputs now to avoid cluttering the code later on out = [] for i in range(33): - out.append(get_spendable_output()) + out.append(self.get_spendable_output()) # Start by building a couple of blocks on top (which output is spent is # in parentheses): # genesis -> b1 (0) -> b2 (1) - block(1, spend=out[0]) - save_spendable_output() - yield accepted() + b1 = self.next_block(1, spend=out[0]) + self.save_spendable_output() - block(2, spend=out[1]) - yield accepted() - save_spendable_output() + b2 = self.next_block(2, spend=out[1]) + self.save_spendable_output() - # so fork like this: + self.sync_blocks([b1, b2]) + + # Fork like this: # # genesis -> b1 (0) -> b2 (1) # \-> b3 (1) # # Nothing should happen at this point. We saw b2 first so it takes priority. - tip(1) - b3 = block(3, spend=out[1]) + self.log.info("Don't reorg to a chain of the same length") + self.move_tip(1) + b3 = self.next_block(3, spend=out[1]) txout_b3 = PreviousSpendableOutput(b3.vtx[1], 0) - yield rejected() - + self.sync_blocks([b3], False) # Now we add another block to make the alternative chain longer. # # genesis -> b1 (0) -> b2 (1) # \-> b3 (1) -> b4 (2) - block(4, spend=out[2]) - yield accepted() - + self.log.info("Reorg to a longer chain") + b4 = self.next_block(4, spend=out[2]) + self.sync_blocks([b4]) # ... and back to the first chain. # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b3 (1) -> b4 (2) - tip(2) - block(5, spend=out[2]) - save_spendable_output() - yield rejected() + self.move_tip(2) + b5 = self.next_block(5, spend=out[2]) + self.save_spendable_output() + self.sync_blocks([b5], False) - block(6, spend=out[3]) - yield accepted() + self.log.info("Reorg back to the original chain") + b6 = self.next_block(6, spend=out[3]) + self.sync_blocks([b6], True) # Try to create a fork that double-spends # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b7 (2) -> b8 (4) # \-> b3 (1) -> b4 (2) - tip(5) - block(7, spend=out[2]) - yield rejected() + self.log.info("Reject a chain with a double spend, even if it is longer") + self.move_tip(5) + b7 = self.next_block(7, spend=out[2]) + self.sync_blocks([b7], False) - block(8, spend=out[4]) - yield rejected() + b8 = self.next_block(8, spend=out[4]) + self.sync_blocks([b8], False, reconnect=True) # Try to create a block that has too much fee # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b9 (4) # \-> b3 (1) -> b4 (2) - tip(6) - block(9, spend=out[4], additional_coinbase_value=1) - yield rejected(RejectResult(16, b'bad-cb-amount')) + self.log.info("Reject a block where the miner creates too much coinbase reward") + self.move_tip(6) + b9 = self.next_block(9, spend=out[4], additional_coinbase_value=1) + self.sync_blocks([b9], False, 16, b'bad-cb-amount', reconnect=True) # Create a fork that ends in a block with too much fee (the one that causes the reorg) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b10 (3) -> b11 (4) # \-> b3 (1) -> b4 (2) - tip(5) - block(10, spend=out[3]) - yield rejected() - - block(11, spend=out[4], additional_coinbase_value=1) - yield rejected(RejectResult(16, b'bad-cb-amount')) + self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer") + self.move_tip(5) + b10 = self.next_block(10, spend=out[3]) + self.sync_blocks([b10], False) + b11 = self.next_block(11, spend=out[4], additional_coinbase_value=1) + self.sync_blocks([b11], False, 16, b'bad-cb-amount', reconnect=True) # Try again, but with a valid fork first # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b14 (5) - # (b12 added last) # \-> b3 (1) -> b4 (2) - tip(5) - b12 = block(12, spend=out[3]) - save_spendable_output() - b13 = block(13, spend=out[4]) - # Deliver the block header for b12, and the block b13. - # b13 should be accepted but the tip won't advance until b12 is delivered. - yield TestInstance([[CBlockHeader(b12), None], [b13, False]]) - - save_spendable_output() - # b14 is invalid, but the node won't know that until it tries to connect - # Tip still can't advance because b12 is missing - block(14, spend=out[5], additional_coinbase_value=1) - yield rejected() - - yield TestInstance([[b12, True, b13.sha256]]) # New tip should be b13. + self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer (on a forked chain)") + self.move_tip(5) + b12 = self.next_block(12, spend=out[3]) + self.save_spendable_output() + b13 = self.next_block(13, spend=out[4]) + self.save_spendable_output() + b14 = self.next_block(14, spend=out[5], additional_coinbase_value=1) + self.sync_blocks([b12, b13, b14], False, 16, b'bad-cb-amount', reconnect=True) + + # New tip should be b13. + assert_equal(node.getbestblockhash(), b13.hash) # Add a block with MAX_BLOCK_SIGOPS and one with one more sigop # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b16 (6) # \-> b3 (1) -> b4 (2) - - # Test that a block with a lot of checksigs is okay + self.log.info("Accept a block with lots of checksigs") lots_of_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS - 1)) - tip(13) - block(15, spend=out[5], script=lots_of_checksigs) - yield accepted() - save_spendable_output() + self.move_tip(13) + b15 = self.next_block(15, spend=out[5], script=lots_of_checksigs) + self.save_spendable_output() + self.sync_blocks([b15], True) - - # Test that a block with too many checksigs is rejected + self.log.info("Reject a block with too many checksigs") too_many_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) - block(16, spend=out[6], script=too_many_checksigs) - yield rejected(RejectResult(16, b'bad-blk-sigops')) - + b16 = self.next_block(16, spend=out[6], script=too_many_checksigs) + self.sync_blocks([b16], False, 16, b'bad-blk-sigops', reconnect=True) # Attempt to spend a transaction created on a different fork # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b17 (b3.vtx[1]) # \-> b3 (1) -> b4 (2) - tip(15) - block(17, spend=txout_b3) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + self.log.info("Reject a block with a spend from a re-org'ed out tx") + self.move_tip(15) + b17 = self.next_block(17, spend=txout_b3) + self.sync_blocks([b17], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a transaction created on a different fork (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) # \-> b18 (b3.vtx[1]) -> b19 (6) # \-> b3 (1) -> b4 (2) - tip(13) - block(18, spend=txout_b3) - yield rejected() + self.log.info("Reject a block with a spend from a re-org'ed out tx (on a forked chain)") + self.move_tip(13) + b18 = self.next_block(18, spend=txout_b3) + self.sync_blocks([b18], False) - block(19, spend=out[6]) - yield rejected() + b19 = self.next_block(19, spend=out[6]) + self.sync_blocks([b19], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a coinbase at depth too low # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b20 (7) # \-> b3 (1) -> b4 (2) - tip(15) - block(20, spend=out[7]) - yield rejected(RejectResult(16, b'bad-txns-premature-spend-of-coinbase')) + self.log.info("Reject a block spending an immature coinbase.") + self.move_tip(15) + b20 = self.next_block(20, spend=out[7]) + self.sync_blocks([b20], False, 16, b'bad-txns-premature-spend-of-coinbase') # Attempt to spend a coinbase at depth too low (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) # \-> b21 (6) -> b22 (5) # \-> b3 (1) -> b4 (2) - tip(13) - block(21, spend=out[6]) - yield rejected() + self.log.info("Reject a block spending an immature coinbase (on a forked chain)") + self.move_tip(13) + b21 = self.next_block(21, spend=out[6]) + self.sync_blocks([b21], False) - block(22, spend=out[5]) - yield rejected() + b22 = self.next_block(22, spend=out[5]) + self.sync_blocks([b22], False, 16, b'bad-txns-premature-spend-of-coinbase') # Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) # \-> b24 (6) -> b25 (7) # \-> b3 (1) -> b4 (2) - tip(15) - b23 = block(23, spend=out[6]) + self.log.info("Accept a block of size MAX_BLOCK_BASE_SIZE") + self.move_tip(15) + b23 = self.next_block(23, spend=out[6]) tx = CTransaction() script_length = MAX_BLOCK_BASE_SIZE - len(b23.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 0))) - b23 = update_block(23, [tx]) + b23 = self.update_block(23, [tx]) # Make sure the math above worked out to produce a max-sized block assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE) - yield accepted() - save_spendable_output() + self.sync_blocks([b23], True) + self.save_spendable_output() - # Make the next block one byte bigger and check that it fails - tip(15) - b24 = block(24, spend=out[6]) + self.log.info("Reject a block of size MAX_BLOCK_BASE_SIZE + 1") + self.move_tip(15) + b24 = self.next_block(24, spend=out[6]) script_length = MAX_BLOCK_BASE_SIZE - len(b24.serialize()) - 69 - script_output = CScript([b'\x00' * (script_length+1)]) + script_output = CScript([b'\x00' * (script_length + 1)]) tx.vout = [CTxOut(0, script_output)] - b24 = update_block(24, [tx]) - assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE+1) - yield rejected(RejectResult(16, b'bad-blk-length')) + b24 = self.update_block(24, [tx]) + assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE + 1) + self.sync_blocks([b24], False, 16, b'bad-blk-length', reconnect=True) - block(25, spend=out[7]) - yield rejected() + b25 = self.next_block(25, spend=out[7]) + self.sync_blocks([b25], False) # Create blocks with a coinbase input script size out of range # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) -> b30 (7) # \-> ... (6) -> ... (7) # \-> b3 (1) -> b4 (2) - tip(15) - b26 = block(26, spend=out[6]) + self.log.info("Reject a block with coinbase input script size out of range") + self.move_tip(15) + b26 = self.next_block(26, spend=out[6]) b26.vtx[0].vin[0].scriptSig = b'\x00' b26.vtx[0].rehash() # update_block causes the merkle root to get updated, even with no new # transactions, and updates the required state. - b26 = update_block(26, []) - yield rejected(RejectResult(16, b'bad-cb-length')) + b26 = self.update_block(26, []) + self.sync_blocks([b26], False, 16, b'bad-cb-length', reconnect=True) # Extend the b26 chain to make sure bitcoind isn't accepting b26 - block(27, spend=out[7]) - yield rejected(False) + b27 = self.next_block(27, spend=out[7]) + self.sync_blocks([b27], False) # Now try a too-large-coinbase script - tip(15) - b28 = block(28, spend=out[6]) + self.move_tip(15) + b28 = self.next_block(28, spend=out[6]) b28.vtx[0].vin[0].scriptSig = b'\x00' * 101 b28.vtx[0].rehash() - b28 = update_block(28, []) - yield rejected(RejectResult(16, b'bad-cb-length')) + b28 = self.update_block(28, []) + self.sync_blocks([b28], False, 16, b'bad-cb-length', reconnect=True) # Extend the b28 chain to make sure bitcoind isn't accepting b28 - block(29, spend=out[7]) - yield rejected(False) + b29 = self.next_block(29, spend=out[7]) + self.sync_blocks([b29], False) # b30 has a max-sized coinbase scriptSig. - tip(23) - b30 = block(30) + self.move_tip(23) + b30 = self.next_block(30) b30.vtx[0].vin[0].scriptSig = b'\x00' * 100 b30.vtx[0].rehash() - b30 = update_block(30, []) - yield accepted() - save_spendable_output() + b30 = self.update_block(30, []) + self.sync_blocks([b30], True) + self.save_spendable_output() # b31 - b35 - check sigops of OP_CHECKMULTISIG / OP_CHECKMULTISIGVERIFY / OP_CHECKSIGVERIFY # @@ -433,42 +347,45 @@ class FullBlockTest(ComparisonTestFramework): # # MULTISIG: each op code counts as 20 sigops. To create the edge case, pack another 19 sigops at the end. - lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19) - b31 = block(31, spend=out[8], script=lots_of_multisigs) + self.log.info("Accept a block with the max number of OP_CHECKMULTISIG sigops") + lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) + b31 = self.next_block(31, spend=out[8], script=lots_of_multisigs) assert_equal(get_legacy_sigopcount_block(b31), MAX_BLOCK_SIGOPS) - yield accepted() - save_spendable_output() + self.sync_blocks([b31], True) + self.save_spendable_output() # this goes over the limit because the coinbase has one sigop + self.log.info("Reject a block with too many OP_CHECKMULTISIG sigops") too_many_multisigs = CScript([OP_CHECKMULTISIG] * (MAX_BLOCK_SIGOPS // 20)) - b32 = block(32, spend=out[9], script=too_many_multisigs) + b32 = self.next_block(32, spend=out[9], script=too_many_multisigs) assert_equal(get_legacy_sigopcount_block(b32), MAX_BLOCK_SIGOPS + 1) - yield rejected(RejectResult(16, b'bad-blk-sigops')) - + self.sync_blocks([b32], False, 16, b'bad-blk-sigops', reconnect=True) # CHECKMULTISIGVERIFY - tip(31) - lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19) - block(33, spend=out[9], script=lots_of_multisigs) - yield accepted() - save_spendable_output() - + self.log.info("Accept a block with the max number of OP_CHECKMULTISIGVERIFY sigops") + self.move_tip(31) + lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) + b33 = self.next_block(33, spend=out[9], script=lots_of_multisigs) + self.sync_blocks([b33], True) + self.save_spendable_output() + + self.log.info("Reject a block with too many OP_CHECKMULTISIGVERIFY sigops") too_many_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * (MAX_BLOCK_SIGOPS // 20)) - block(34, spend=out[10], script=too_many_multisigs) - yield rejected(RejectResult(16, b'bad-blk-sigops')) - + b34 = self.next_block(34, spend=out[10], script=too_many_multisigs) + self.sync_blocks([b34], False, 16, b'bad-blk-sigops', reconnect=True) # CHECKSIGVERIFY - tip(33) + self.log.info("Accept a block with the max number of OP_CHECKSIGVERIFY sigops") + self.move_tip(33) lots_of_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS - 1)) - b35 = block(35, spend=out[10], script=lots_of_checksigs) - yield accepted() - save_spendable_output() + b35 = self.next_block(35, spend=out[10], script=lots_of_checksigs) + self.sync_blocks([b35], True) + self.save_spendable_output() + self.log.info("Reject a block with too many OP_CHECKSIGVERIFY sigops") too_many_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS)) - block(36, spend=out[11], script=too_many_checksigs) - yield rejected(RejectResult(16, b'bad-blk-sigops')) - + b36 = self.next_block(36, spend=out[11], script=too_many_checksigs) + self.sync_blocks([b36], False, 16, b'bad-blk-sigops', reconnect=True) # Check spending of a transaction in a block which failed to connect # @@ -479,17 +396,18 @@ class FullBlockTest(ComparisonTestFramework): # # save 37's spendable output, but then double-spend out11 to invalidate the block - tip(35) - b37 = block(37, spend=out[11]) + self.log.info("Reject a block spending transaction from a block which failed to connect") + self.move_tip(35) + b37 = self.next_block(37, spend=out[11]) txout_b37 = PreviousSpendableOutput(b37.vtx[1], 0) - tx = create_and_sign_tx(out[11].tx, out[11].n, 0) - b37 = update_block(37, [tx]) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + tx = self.create_and_sign_transaction(out[11].tx, out[11].n, 0) + b37 = self.update_block(37, [tx]) + self.sync_blocks([b37], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # attempt to spend b37's first non-coinbase tx, at which point b37 was still considered valid - tip(35) - block(38, spend=txout_b37) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + self.move_tip(35) + b38 = self.next_block(38, spend=txout_b37) + self.sync_blocks([b38], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # Check P2SH SigOp counting # @@ -502,45 +420,45 @@ class FullBlockTest(ComparisonTestFramework): # redeem_script = COINBASE_PUBKEY, (OP_2DUP+OP_CHECKSIGVERIFY) * 5, OP_CHECKSIG # p2sh_script = OP_HASH160, ripemd160(sha256(script)), OP_EQUAL # - tip(35) - b39 = block(39) + self.log.info("Check P2SH SIGOPS are correctly counted") + self.move_tip(35) + b39 = self.next_block(39) b39_outputs = 0 b39_sigops_per_output = 6 # Build the redeem script, hash it, use hash to create the p2sh script - redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY]*5 + [OP_CHECKSIG]) + redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY] * 5 + [OP_CHECKSIG]) redeem_script_hash = hash160(redeem_script) p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL]) # Create a transaction that spends one satoshi to the p2sh_script, the rest to OP_TRUE # This must be signed because it is spending a coinbase spend = out[11] - tx = create_tx(spend.tx, spend.n, 1, p2sh_script) + tx = self.create_tx(spend.tx, spend.n, 1, p2sh_script) tx.vout.append(CTxOut(spend.tx.vout[spend.n].nValue - 1, CScript([OP_TRUE]))) self.sign_tx(tx, spend.tx, spend.n) tx.rehash() - b39 = update_block(39, [tx]) + b39 = self.update_block(39, [tx]) b39_outputs += 1 # Until block is full, add tx's with 1 satoshi to p2sh_script, the rest to OP_TRUE tx_new = None tx_last = tx - total_size=len(b39.serialize()) + total_size = len(b39.serialize()) while(total_size < MAX_BLOCK_BASE_SIZE): - tx_new = create_tx(tx_last, 1, 1, p2sh_script) + tx_new = self.create_tx(tx_last, 1, 1, p2sh_script) tx_new.vout.append(CTxOut(tx_last.vout[1].nValue - 1, CScript([OP_TRUE]))) tx_new.rehash() total_size += len(tx_new.serialize()) if total_size >= MAX_BLOCK_BASE_SIZE: break - b39.vtx.append(tx_new) # add tx to block + b39.vtx.append(tx_new) # add tx to block tx_last = tx_new b39_outputs += 1 - b39 = update_block(39, []) - yield accepted() - save_spendable_output() - + b39 = self.update_block(39, []) + self.sync_blocks([b39], True) + self.save_spendable_output() # Test sigops in P2SH redeem scripts # @@ -549,15 +467,16 @@ class FullBlockTest(ComparisonTestFramework): # # b41 does the same, less one, so it has the maximum sigops permitted. # - tip(39) - b40 = block(40, spend=out[12]) + self.log.info("Reject a block with too many P2SH sigops") + self.move_tip(39) + b40 = self.next_block(40, spend=out[12]) sigops = get_legacy_sigopcount_block(b40) numTxes = (MAX_BLOCK_SIGOPS - sigops) // b39_sigops_per_output assert_equal(numTxes <= b39_outputs, True) lastOutpoint = COutPoint(b40.vtx[1].sha256, 0) new_txs = [] - for i in range(1, numTxes+1): + for i in range(1, numTxes + 1): tx = CTransaction() tx.vout.append(CTxOut(1, CScript([OP_TRUE]))) tx.vin.append(CTxIn(lastOutpoint, b'')) @@ -579,35 +498,34 @@ class FullBlockTest(ComparisonTestFramework): tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b40_sigops_to_fill))) tx.rehash() new_txs.append(tx) - update_block(40, new_txs) - yield rejected(RejectResult(16, b'bad-blk-sigops')) + self.update_block(40, new_txs) + self.sync_blocks([b40], False, 16, b'bad-blk-sigops', reconnect=True) # same as b40, but one less sigop - tip(39) - block(41, spend=None) - update_block(41, b40.vtx[1:-1]) + self.log.info("Accept a block with the max number of P2SH sigops") + self.move_tip(39) + b41 = self.next_block(41, spend=None) + self.update_block(41, b40.vtx[1:-1]) b41_sigops_to_fill = b40_sigops_to_fill - 1 tx = CTransaction() tx.vin.append(CTxIn(lastOutpoint, b'')) tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b41_sigops_to_fill))) tx.rehash() - update_block(41, [tx]) - yield accepted() + self.update_block(41, [tx]) + self.sync_blocks([b41], True) # Fork off of b39 to create a constant base again # # b23 (6) -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) # \-> b41 (12) # - tip(39) - block(42, spend=out[12]) - yield rejected() - save_spendable_output() - - block(43, spend=out[13]) - yield accepted() - save_spendable_output() + self.move_tip(39) + b42 = self.next_block(42, spend=out[12]) + self.save_spendable_output() + b43 = self.next_block(43, spend=out[13]) + self.save_spendable_output() + self.sync_blocks([b42, b43], True) # Test a number of really invalid scenarios # @@ -616,6 +534,7 @@ class FullBlockTest(ComparisonTestFramework): # The next few blocks are going to be created "by hand" since they'll do funky things, such as having # the first transaction be non-coinbase, etc. The purpose of b44 is to make sure this works. + self.log.info("Build block 44 manually") height = self.block_heights[self.tip.sha256] + 1 coinbase = create_coinbase(height, self.coinbase_pubkey) b44 = CBlock() @@ -628,10 +547,10 @@ class FullBlockTest(ComparisonTestFramework): self.tip = b44 self.block_heights[b44.sha256] = height self.blocks[44] = b44 - yield accepted() + self.sync_blocks([b44], True) - # A block with a non-coinbase as the first tx - non_coinbase = create_tx(out[15].tx, out[15].n, 1) + self.log.info("Reject a block with a non-coinbase as the first tx") + non_coinbase = self.create_tx(out[15].tx, out[15].n, 1) b45 = CBlock() b45.nTime = self.tip.nTime + 1 b45.hashPrevBlock = self.tip.sha256 @@ -640,104 +559,102 @@ class FullBlockTest(ComparisonTestFramework): b45.hashMerkleRoot = b45.calc_merkle_root() b45.calc_sha256() b45.solve() - self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256]+1 + self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256] + 1 self.tip = b45 self.blocks[45] = b45 - yield rejected(RejectResult(16, b'bad-cb-missing')) + self.sync_blocks([b45], False, 16, b'bad-cb-missing', reconnect=True) - # A block with no txns - tip(44) + self.log.info("Reject a block with no transactions") + self.move_tip(44) b46 = CBlock() - b46.nTime = b44.nTime+1 + b46.nTime = b44.nTime + 1 b46.hashPrevBlock = b44.sha256 b46.nBits = 0x207fffff b46.vtx = [] b46.hashMerkleRoot = 0 b46.solve() - self.block_heights[b46.sha256] = self.block_heights[b44.sha256]+1 + self.block_heights[b46.sha256] = self.block_heights[b44.sha256] + 1 self.tip = b46 assert 46 not in self.blocks self.blocks[46] = b46 - s = ser_uint256(b46.hashMerkleRoot) - yield rejected(RejectResult(16, b'bad-blk-length')) + self.sync_blocks([b46], False, 16, b'bad-blk-length', reconnect=True) - # A block with invalid work - tip(44) - b47 = block(47, solve=False) + self.log.info("Reject a block with invalid work") + self.move_tip(44) + b47 = self.next_block(47, solve=False) target = uint256_from_compact(b47.nBits) - while b47.sha256 < target: #changed > to < + while b47.sha256 < target: b47.nNonce += 1 b47.rehash() - yield rejected(RejectResult(16, b'high-hash')) + self.sync_blocks([b47], False, request_block=False) - # A block with timestamp > 2 hrs in the future - tip(44) - b48 = block(48, solve=False) + self.log.info("Reject a block with a timestamp >2 hours in the future") + self.move_tip(44) + b48 = self.next_block(48, solve=False) b48.nTime = int(time.time()) + 60 * 60 * 3 b48.solve() - yield rejected(RejectResult(16, b'time-too-new')) + self.sync_blocks([b48], False, request_block=False) - # A block with an invalid merkle hash - tip(44) - b49 = block(49) + self.log.info("Reject a block with invalid merkle hash") + self.move_tip(44) + b49 = self.next_block(49) b49.hashMerkleRoot += 1 b49.solve() - yield rejected(RejectResult(16, b'bad-txnmrklroot')) + self.sync_blocks([b49], False, 16, b'bad-txnmrklroot', reconnect=True) - # A block with an incorrect POW limit - tip(44) - b50 = block(50) + self.log.info("Reject a block with incorrect POW limit") + self.move_tip(44) + b50 = self.next_block(50) b50.nBits = b50.nBits - 1 b50.solve() - yield rejected(RejectResult(16, b'bad-diffbits')) + self.sync_blocks([b50], False, request_block=False, reconnect=True) - # A block with two coinbase txns - tip(44) - b51 = block(51) + self.log.info("Reject a block with two coinbase transactions") + self.move_tip(44) + b51 = self.next_block(51) cb2 = create_coinbase(51, self.coinbase_pubkey) - b51 = update_block(51, [cb2]) - yield rejected(RejectResult(16, b'bad-cb-multiple')) + b51 = self.update_block(51, [cb2]) + self.sync_blocks([b51], False, 16, b'bad-cb-multiple', reconnect=True) - # A block w/ duplicate txns + self.log.info("Reject a block with duplicate transactions") # Note: txns have to be in the right position in the merkle tree to trigger this error - tip(44) - b52 = block(52, spend=out[15]) - tx = create_tx(b52.vtx[1], 0, 1) - b52 = update_block(52, [tx, tx]) - yield rejected(RejectResult(16, b'bad-txns-duplicate')) + self.move_tip(44) + b52 = self.next_block(52, spend=out[15]) + tx = self.create_tx(b52.vtx[1], 0, 1) + b52 = self.update_block(52, [tx, tx]) + self.sync_blocks([b52], False, 16, b'bad-txns-duplicate', reconnect=True) # Test block timestamps # -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) # \-> b54 (15) # - tip(43) - block(53, spend=out[14]) - yield rejected() # rejected since b44 is at same height - save_spendable_output() + self.move_tip(43) + b53 = self.next_block(53, spend=out[14]) + self.sync_blocks([b53], False) + self.save_spendable_output() - # invalid timestamp (b35 is 5 blocks back, so its time is MedianTimePast) - b54 = block(54, spend=out[15]) + self.log.info("Reject a block with timestamp before MedianTimePast") + b54 = self.next_block(54, spend=out[15]) b54.nTime = b35.nTime - 1 b54.solve() - yield rejected(RejectResult(16, b'time-too-old')) + self.sync_blocks([b54], False, request_block=False) # valid timestamp - tip(53) - b55 = block(55, spend=out[15]) + self.move_tip(53) + b55 = self.next_block(55, spend=out[15]) b55.nTime = b35.nTime - update_block(55, []) - yield accepted() - save_spendable_output() + self.update_block(55, []) + self.sync_blocks([b55], True) + self.save_spendable_output() - - # Test CVE-2012-2459 + # Test Merkle tree malleability # # -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57p2 (16) # \-> b57 (16) # \-> b56p2 (16) # \-> b56 (16) # - # Merkle tree malleability (CVE-2012-2459): repeating sequences of transactions in a block without + # Merkle tree malleability (CVE-2012-2459): repeating sequences of transactions in a block without # affecting the merkle root of a block, while still invalidating it. # See: src/consensus/merkle.h # @@ -758,46 +675,48 @@ class FullBlockTest(ComparisonTestFramework): # that the error was caught early, avoiding a DOS vulnerability.) # b57 - a good block with 2 txs, don't submit until end - tip(55) - b57 = block(57) - tx = create_and_sign_tx(out[16].tx, out[16].n, 1) - tx1 = create_tx(tx, 0, 1) - b57 = update_block(57, [tx, tx1]) + self.move_tip(55) + b57 = self.next_block(57) + tx = self.create_and_sign_transaction(out[16].tx, out[16].n, 1) + tx1 = self.create_tx(tx, 0, 1) + b57 = self.update_block(57, [tx, tx1]) # b56 - copy b57, add a duplicate tx - tip(55) + self.log.info("Reject a block with a duplicate transaction in the Merkle Tree (but with a valid Merkle Root)") + self.move_tip(55) b56 = copy.deepcopy(b57) self.blocks[56] = b56 - assert_equal(len(b56.vtx),3) - b56 = update_block(56, [tx1]) + assert_equal(len(b56.vtx), 3) + b56 = self.update_block(56, [tx1]) assert_equal(b56.hash, b57.hash) - yield rejected(RejectResult(16, b'bad-txns-duplicate')) + self.sync_blocks([b56], False, 16, b'bad-txns-duplicate', reconnect=True) # b57p2 - a good block with 6 tx'es, don't submit until end - tip(55) - b57p2 = block("57p2") - tx = create_and_sign_tx(out[16].tx, out[16].n, 1) - tx1 = create_tx(tx, 0, 1) - tx2 = create_tx(tx1, 0, 1) - tx3 = create_tx(tx2, 0, 1) - tx4 = create_tx(tx3, 0, 1) - b57p2 = update_block("57p2", [tx, tx1, tx2, tx3, tx4]) + self.move_tip(55) + b57p2 = self.next_block("57p2") + tx = self.create_and_sign_transaction(out[16].tx, out[16].n, 1) + tx1 = self.create_tx(tx, 0, 1) + tx2 = self.create_tx(tx1, 0, 1) + tx3 = self.create_tx(tx2, 0, 1) + tx4 = self.create_tx(tx3, 0, 1) + b57p2 = self.update_block("57p2", [tx, tx1, tx2, tx3, tx4]) # b56p2 - copy b57p2, duplicate two non-consecutive tx's - tip(55) + self.log.info("Reject a block with two duplicate transactions in the Merkle Tree (but with a valid Merkle Root)") + self.move_tip(55) b56p2 = copy.deepcopy(b57p2) self.blocks["b56p2"] = b56p2 assert_equal(b56p2.hash, b57p2.hash) - assert_equal(len(b56p2.vtx),6) - b56p2 = update_block("b56p2", [tx3, tx4]) - yield rejected(RejectResult(16, b'bad-txns-duplicate')) + assert_equal(len(b56p2.vtx), 6) + b56p2 = self.update_block("b56p2", [tx3, tx4]) + self.sync_blocks([b56p2], False, 16, b'bad-txns-duplicate', reconnect=True) - tip("57p2") - yield accepted() + self.move_tip("57p2") + self.sync_blocks([b57p2], True) - tip(57) - yield rejected() #rejected because 57p2 seen first - save_spendable_output() + self.move_tip(57) + self.sync_blocks([b57], False) # The tip is not updated because 57p2 seen first + self.save_spendable_output() # Test a few invalid tx types # @@ -806,28 +725,30 @@ class FullBlockTest(ComparisonTestFramework): # # tx with prevout.n out of range - tip(57) - b58 = block(58, spend=out[17]) + self.log.info("Reject a block with a transaction with prevout.n out of range") + self.move_tip(57) + b58 = self.next_block(58, spend=out[17]) tx = CTransaction() assert(len(out[17].tx.vout) < 42) tx.vin.append(CTxIn(COutPoint(out[17].tx.sha256, 42), CScript([OP_TRUE]), 0xffffffff)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() - b58 = update_block(58, [tx]) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + b58 = self.update_block(58, [tx]) + self.sync_blocks([b58], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) - # tx with output value > input value out of range - tip(57) - b59 = block(59) - tx = create_and_sign_tx(out[17].tx, out[17].n, 51*COIN) - b59 = update_block(59, [tx]) - yield rejected(RejectResult(16, b'bad-txns-in-belowout')) + # tx with output value > input value + self.log.info("Reject a block with a transaction with outputs > inputs") + self.move_tip(57) + b59 = self.next_block(59) + tx = self.create_and_sign_transaction(out[17].tx, out[17].n, 51 * COIN) + b59 = self.update_block(59, [tx]) + self.sync_blocks([b59], False, 16, b'bad-txns-in-belowout', reconnect=True) # reset to good chain - tip(57) - b60 = block(60, spend=out[17]) - yield accepted() - save_spendable_output() + self.move_tip(57) + b60 = self.next_block(60, spend=out[17]) + self.sync_blocks([b60], True) + self.save_spendable_output() # Test BIP30 # @@ -838,46 +759,46 @@ class FullBlockTest(ComparisonTestFramework): # not-fully-spent transaction in the same chain. To test, make identical coinbases; # the second one should be rejected. # - tip(60) - b61 = block(61, spend=out[18]) - b61.vtx[0].vin[0].scriptSig = b60.vtx[0].vin[0].scriptSig #equalize the coinbases + self.log.info("Reject a block with a transaction with a duplicate hash of a previous transaction (BIP30)") + self.move_tip(60) + b61 = self.next_block(61, spend=out[18]) + b61.vtx[0].vin[0].scriptSig = b60.vtx[0].vin[0].scriptSig # Equalize the coinbases b61.vtx[0].rehash() - b61 = update_block(61, []) + b61 = self.update_block(61, []) assert_equal(b60.vtx[0].serialize(), b61.vtx[0].serialize()) - yield rejected(RejectResult(16, b'bad-txns-BIP30')) - + self.sync_blocks([b61], False, 16, b'bad-txns-BIP30', reconnect=True) # Test tx.isFinal is properly rejected (not an exhaustive tx.isFinal test, that should be in data-driven transaction tests) # # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) # \-> b62 (18) # - tip(60) - b62 = block(62) + self.log.info("Reject a block with a transaction with a nonfinal locktime") + self.move_tip(60) + b62 = self.next_block(62) tx = CTransaction() - tx.nLockTime = 0xffffffff #this locktime is non-final + tx.nLockTime = 0xffffffff # this locktime is non-final assert(out[18].n < len(out[18].tx.vout)) - tx.vin.append(CTxIn(COutPoint(out[18].tx.sha256, out[18].n))) # don't set nSequence + tx.vin.append(CTxIn(COutPoint(out[18].tx.sha256, out[18].n))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) assert(tx.vin[0].nSequence < 0xffffffff) tx.calc_sha256() - b62 = update_block(62, [tx]) - yield rejected(RejectResult(16, b'bad-txns-nonfinal')) - + b62 = self.update_block(62, [tx]) + self.sync_blocks([b62], False, 16, b'bad-txns-nonfinal') # Test a non-final coinbase is also rejected # # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) # \-> b63 (-) # - tip(60) - b63 = block(63) + self.log.info("Reject a block with a coinbase transaction with a nonfinal locktime") + self.move_tip(60) + b63 = self.next_block(63) b63.vtx[0].nLockTime = 0xffffffff b63.vtx[0].vin[0].nSequence = 0xDEADBEEF b63.vtx[0].rehash() - b63 = update_block(63, []) - yield rejected(RejectResult(16, b'bad-txns-nonfinal')) - + b63 = self.update_block(63, []) + self.sync_blocks([b63], False, 16, b'bad-txns-nonfinal') # This checks that a block with a bloated VARINT between the block_header and the array of tx such that # the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint, @@ -893,8 +814,9 @@ class FullBlockTest(ComparisonTestFramework): # b64a is a bloated block (non-canonical varint) # b64 is a good block (same as b64 but w/ canonical varint) # - tip(60) - regular_block = block("64a", spend=out[18]) + self.log.info("Accept a valid block even if a bloated version of the block has previously been sent") + self.move_tip(60) + regular_block = self.next_block("64a", spend=out[18]) # make it a "broken_block," with non-canonical serialization b64a = CBrokenBlock(regular_block) @@ -908,45 +830,51 @@ class FullBlockTest(ComparisonTestFramework): script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) - b64a = update_block("64a", [tx]) + b64a = self.update_block("64a", [tx]) assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) - yield TestInstance([[self.tip, None]]) + self.sync_blocks([b64a], False, 1, b'error parsing message') - # comptool workaround: to make sure b64 is delivered, manually erase b64a from blockstore - self.test.block_store.erase(b64a.sha256) + # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently + # resend the header message, it won't send us the getdata message again. Just + # disconnect and reconnect and then call sync_blocks. + # TODO: improve this test to be less dependent on P2P DOS behaviour. + node.disconnect_p2ps() + self.reconnect_p2p() - tip(60) + self.move_tip(60) b64 = CBlock(b64a) b64.vtx = copy.deepcopy(b64a.vtx) assert_equal(b64.hash, b64a.hash) assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE) self.blocks[64] = b64 - update_block(64, []) - yield accepted() - save_spendable_output() + b64 = self.update_block(64, []) + self.sync_blocks([b64], True) + self.save_spendable_output() # Spend an output created in the block itself # # -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) # - tip(64) - block(65) - tx1 = create_and_sign_tx(out[19].tx, out[19].n, out[19].tx.vout[0].nValue) - tx2 = create_and_sign_tx(tx1, 0, 0) - update_block(65, [tx1, tx2]) - yield accepted() - save_spendable_output() + self.log.info("Accept a block with a transaction spending an output created in the same block") + self.move_tip(64) + b65 = self.next_block(65) + tx1 = self.create_and_sign_transaction(out[19].tx, out[19].n, out[19].tx.vout[0].nValue) + tx2 = self.create_and_sign_transaction(tx1, 0, 0) + b65 = self.update_block(65, [tx1, tx2]) + self.sync_blocks([b65], True) + self.save_spendable_output() # Attempt to spend an output created later in the same block # # -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) # \-> b66 (20) - tip(65) - block(66) - tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) - tx2 = create_and_sign_tx(tx1, 0, 1) - update_block(66, [tx2, tx1]) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + self.log.info("Reject a block with a transaction spending an output created later in the same block") + self.move_tip(65) + b66 = self.next_block(66) + tx1 = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) + tx2 = self.create_and_sign_transaction(tx1, 0, 1) + b66 = self.update_block(66, [tx2, tx1]) + self.sync_blocks([b66], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # Attempt to double-spend a transaction created in a block # @@ -954,13 +882,14 @@ class FullBlockTest(ComparisonTestFramework): # \-> b67 (20) # # - tip(65) - block(67) - tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) - tx2 = create_and_sign_tx(tx1, 0, 1) - tx3 = create_and_sign_tx(tx1, 0, 2) - update_block(67, [tx1, tx2, tx3]) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + self.log.info("Reject a block with a transaction double spending a transaction creted in the same block") + self.move_tip(65) + b67 = self.next_block(67) + tx1 = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) + tx2 = self.create_and_sign_transaction(tx1, 0, 1) + tx3 = self.create_and_sign_transaction(tx1, 0, 2) + b67 = self.update_block(67, [tx1, tx2, tx3]) + self.sync_blocks([b67], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # More tests of block subsidy # @@ -974,34 +903,36 @@ class FullBlockTest(ComparisonTestFramework): # b69 - coinbase with extra 10 satoshis, and a tx that gives a 10 satoshi fee # this succeeds # - tip(65) - block(68, additional_coinbase_value=10) - tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-9) - update_block(68, [tx]) - yield rejected(RejectResult(16, b'bad-cb-amount')) - - tip(65) - b69 = block(69, additional_coinbase_value=10) - tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-10) - update_block(69, [tx]) - yield accepted() - save_spendable_output() + self.log.info("Reject a block trying to claim too much subsidy in the coinbase transaction") + self.move_tip(65) + b68 = self.next_block(68, additional_coinbase_value=10) + tx = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue - 9) + b68 = self.update_block(68, [tx]) + self.sync_blocks([b68], False, 16, b'bad-cb-amount', reconnect=True) + + self.log.info("Accept a block claiming the correct subsidy in the coinbase transaction") + self.move_tip(65) + b69 = self.next_block(69, additional_coinbase_value=10) + tx = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue - 10) + self.update_block(69, [tx]) + self.sync_blocks([b69], True) + self.save_spendable_output() # Test spending the outpoint of a non-existent transaction # # -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) # \-> b70 (21) # - tip(69) - block(70, spend=out[21]) + self.log.info("Reject a block containing a transaction spending from a non-existent input") + self.move_tip(69) + b70 = self.next_block(70, spend=out[21]) bogus_tx = CTransaction() bogus_tx.sha256 = uint256_from_str(b"23c70ed7c0506e9178fc1a987f40a33946d4ad4c962b5ae3a52546da53af0c5c") tx = CTransaction() tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff)) tx.vout.append(CTxOut(1, b"")) - update_block(70, [tx]) - yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) - + b70 = self.update_block(70, [tx]) + self.sync_blocks([b70], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) # Test accepting an invalid block which has the same hash as a valid one (via merkle tree tricks) # @@ -1009,13 +940,13 @@ class FullBlockTest(ComparisonTestFramework): # \-> b71 (21) # # b72 is a good block. - # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b71. - # - tip(69) - b72 = block(72) - tx1 = create_and_sign_tx(out[21].tx, out[21].n, 2) - tx2 = create_and_sign_tx(tx1, 0, 1) - b72 = update_block(72, [tx1, tx2]) # now tip is 72 + # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b72. + self.log.info("Reject a block containing a duplicate transaction but with the same Merkle root (Merkle tree malleability") + self.move_tip(69) + b72 = self.next_block(72) + tx1 = self.create_and_sign_transaction(out[21].tx, out[21].n, 2) + tx2 = self.create_and_sign_transaction(tx1, 0, 1) + b72 = self.update_block(72, [tx1, tx2]) # now tip is 72 b71 = copy.deepcopy(b72) b71.vtx.append(tx2) # add duplicate tx2 self.block_heights[b71.sha256] = self.block_heights[b69.sha256] + 1 # b71 builds off b69 @@ -1025,12 +956,12 @@ class FullBlockTest(ComparisonTestFramework): assert_equal(len(b72.vtx), 3) assert_equal(b72.sha256, b71.sha256) - tip(71) - yield rejected(RejectResult(16, b'bad-txns-duplicate')) - tip(72) - yield accepted() - save_spendable_output() + self.move_tip(71) + self.sync_blocks([b71], False, 16, b'bad-txns-duplicate', reconnect=True) + self.move_tip(72) + self.sync_blocks([b72], True) + self.save_spendable_output() # Test some invalid scripts and MAX_BLOCK_SIGOPS # @@ -1048,23 +979,23 @@ class FullBlockTest(ComparisonTestFramework): # bytearray[20,000-20,003]: 521 (max_script_element_size+1, in little-endian format) # bytearray[20,004-20,525]: unread data (script_element) # bytearray[20,526] : OP_CHECKSIG (this puts us over the limit) - # - tip(72) - b73 = block(73) + self.log.info("Reject a block containing too many sigops after a large script element") + self.move_tip(72) + b73 = self.next_block(73) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 + 1 a = bytearray([OP_CHECKSIG] * size) - a[MAX_BLOCK_SIGOPS - 1] = int("4e",16) # OP_PUSHDATA4 + a[MAX_BLOCK_SIGOPS - 1] = int("4e", 16) # OP_PUSHDATA4 element_size = MAX_SCRIPT_ELEMENT_SIZE + 1 a[MAX_BLOCK_SIGOPS] = element_size % 256 - a[MAX_BLOCK_SIGOPS+1] = element_size // 256 - a[MAX_BLOCK_SIGOPS+2] = 0 - a[MAX_BLOCK_SIGOPS+3] = 0 + a[MAX_BLOCK_SIGOPS + 1] = element_size // 256 + a[MAX_BLOCK_SIGOPS + 2] = 0 + a[MAX_BLOCK_SIGOPS + 3] = 0 - tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) - b73 = update_block(73, [tx]) - assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS+1) - yield rejected(RejectResult(16, b'bad-blk-sigops')) + tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a)) + b73 = self.update_block(73, [tx]) + assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS + 1) + self.sync_blocks([b73], False, 16, b'bad-blk-sigops', reconnect=True) # b74/75 - if we push an invalid script element, all prevous sigops are counted, # but sigops after the element are not counted. @@ -1076,45 +1007,44 @@ class FullBlockTest(ComparisonTestFramework): # # b74 fails because we put MAX_BLOCK_SIGOPS+1 before the element # b75 succeeds because we put MAX_BLOCK_SIGOPS before the element - # - # - tip(72) - b74 = block(74) - size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561 + self.log.info("Check sigops are counted correctly after an invalid script element") + self.move_tip(72) + b74 = self.next_block(74) + size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561 a = bytearray([OP_CHECKSIG] * size) a[MAX_BLOCK_SIGOPS] = 0x4e - a[MAX_BLOCK_SIGOPS+1] = 0xfe - a[MAX_BLOCK_SIGOPS+2] = 0xff - a[MAX_BLOCK_SIGOPS+3] = 0xff - a[MAX_BLOCK_SIGOPS+4] = 0xff - tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) - b74 = update_block(74, [tx]) - yield rejected(RejectResult(16, b'bad-blk-sigops')) - - tip(72) - b75 = block(75) + a[MAX_BLOCK_SIGOPS + 1] = 0xfe + a[MAX_BLOCK_SIGOPS + 2] = 0xff + a[MAX_BLOCK_SIGOPS + 3] = 0xff + a[MAX_BLOCK_SIGOPS + 4] = 0xff + tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a)) + b74 = self.update_block(74, [tx]) + self.sync_blocks([b74], False, 16, b'bad-blk-sigops', reconnect=True) + + self.move_tip(72) + b75 = self.next_block(75) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 a = bytearray([OP_CHECKSIG] * size) - a[MAX_BLOCK_SIGOPS-1] = 0x4e + a[MAX_BLOCK_SIGOPS - 1] = 0x4e a[MAX_BLOCK_SIGOPS] = 0xff - a[MAX_BLOCK_SIGOPS+1] = 0xff - a[MAX_BLOCK_SIGOPS+2] = 0xff - a[MAX_BLOCK_SIGOPS+3] = 0xff - tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) - b75 = update_block(75, [tx]) - yield accepted() - save_spendable_output() + a[MAX_BLOCK_SIGOPS + 1] = 0xff + a[MAX_BLOCK_SIGOPS + 2] = 0xff + a[MAX_BLOCK_SIGOPS + 3] = 0xff + tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a)) + b75 = self.update_block(75, [tx]) + self.sync_blocks([b75], True) + self.save_spendable_output() # Check that if we push an element filled with CHECKSIGs, they are not counted - tip(75) - b76 = block(76) + self.move_tip(75) + b76 = self.next_block(76) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 a = bytearray([OP_CHECKSIG] * size) - a[MAX_BLOCK_SIGOPS-1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs - tx = create_and_sign_tx(out[23].tx, 0, 1, CScript(a)) - b76 = update_block(76, [tx]) - yield accepted() - save_spendable_output() + a[MAX_BLOCK_SIGOPS - 1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs + tx = self.create_and_sign_transaction(out[23].tx, 0, 1, CScript(a)) + b76 = self.update_block(76, [tx]) + self.sync_blocks([b76], True) + self.save_spendable_output() # Test transaction resurrection # @@ -1133,39 +1063,39 @@ class FullBlockTest(ComparisonTestFramework): # To get around this issue, we construct transactions which are not signed and which # spend to OP_TRUE. If the standard-ness rules change, this test would need to be # updated. (Perhaps to spend to a P2SH OP_TRUE script) - # - tip(76) - block(77) - tx77 = create_and_sign_tx(out[24].tx, out[24].n, 10*COIN) - update_block(77, [tx77]) - yield accepted() - save_spendable_output() - - block(78) - tx78 = create_tx(tx77, 0, 9*COIN) - update_block(78, [tx78]) - yield accepted() - - block(79) - tx79 = create_tx(tx78, 0, 8*COIN) - update_block(79, [tx79]) - yield accepted() + self.log.info("Test transaction resurrection during a re-org") + self.move_tip(76) + b77 = self.next_block(77) + tx77 = self.create_and_sign_transaction(out[24].tx, out[24].n, 10 * COIN) + b77 = self.update_block(77, [tx77]) + self.sync_blocks([b77], True) + self.save_spendable_output() + + b78 = self.next_block(78) + tx78 = self.create_tx(tx77, 0, 9 * COIN) + b78 = self.update_block(78, [tx78]) + self.sync_blocks([b78], True) + + b79 = self.next_block(79) + tx79 = self.create_tx(tx78, 0, 8 * COIN) + b79 = self.update_block(79, [tx79]) + self.sync_blocks([b79], True) # mempool should be empty assert_equal(len(self.nodes[0].getrawmempool()), 0) - tip(77) - block(80, spend=out[25]) - yield rejected() - save_spendable_output() + self.move_tip(77) + b80 = self.next_block(80, spend=out[25]) + self.sync_blocks([b80], False, request_block=False) + self.save_spendable_output() - block(81, spend=out[26]) - yield rejected() # other chain is same length - save_spendable_output() + b81 = self.next_block(81, spend=out[26]) + self.sync_blocks([b81], False, request_block=False) # other chain is same length + self.save_spendable_output() - block(82, spend=out[27]) - yield accepted() # now this chain is longer, triggers re-org - save_spendable_output() + b82 = self.next_block(82, spend=out[27]) + self.sync_blocks([b82], True) # now this chain is longer, triggers re-org + self.save_spendable_output() # now check that tx78 and tx79 have been put back into the peer's mempool mempool = self.nodes[0].getrawmempool() @@ -1173,33 +1103,32 @@ class FullBlockTest(ComparisonTestFramework): assert(tx78.hash in mempool) assert(tx79.hash in mempool) - # Test invalid opcodes in dead execution paths. # # -> b81 (26) -> b82 (27) -> b83 (28) # - block(83) + self.log.info("Accept a block with invalid opcodes in dead execution paths") + b83 = self.next_block(83) op_codes = [OP_IF, OP_INVALIDOPCODE, OP_ELSE, OP_TRUE, OP_ENDIF] script = CScript(op_codes) - tx1 = create_and_sign_tx(out[28].tx, out[28].n, out[28].tx.vout[0].nValue, script) + tx1 = self.create_and_sign_transaction(out[28].tx, out[28].n, out[28].tx.vout[0].nValue, script) - tx2 = create_and_sign_tx(tx1, 0, 0, CScript([OP_TRUE])) + tx2 = self.create_and_sign_transaction(tx1, 0, 0, CScript([OP_TRUE])) tx2.vin[0].scriptSig = CScript([OP_FALSE]) tx2.rehash() - update_block(83, [tx1, tx2]) - yield accepted() - save_spendable_output() - + b83 = self.update_block(83, [tx1, tx2]) + self.sync_blocks([b83], True) + self.save_spendable_output() # Reorg on/off blocks that have OP_RETURN in them (and try to spend them) # # -> b81 (26) -> b82 (27) -> b83 (28) -> b84 (29) -> b87 (30) -> b88 (31) # \-> b85 (29) -> b86 (30) \-> b89a (32) # - # - block(84) - tx1 = create_tx(out[29].tx, out[29].n, 0, CScript([OP_RETURN])) + self.log.info("Test re-orging blocks with OP_RETURN in them") + b84 = self.next_block(84) + tx1 = self.create_tx(out[29].tx, out[29].n, 0, CScript([OP_RETURN])) tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) @@ -1207,87 +1136,186 @@ class FullBlockTest(ComparisonTestFramework): tx1.calc_sha256() self.sign_tx(tx1, out[29].tx, out[29].n) tx1.rehash() - tx2 = create_tx(tx1, 1, 0, CScript([OP_RETURN])) + tx2 = self.create_tx(tx1, 1, 0, CScript([OP_RETURN])) tx2.vout.append(CTxOut(0, CScript([OP_RETURN]))) - tx3 = create_tx(tx1, 2, 0, CScript([OP_RETURN])) + tx3 = self.create_tx(tx1, 2, 0, CScript([OP_RETURN])) tx3.vout.append(CTxOut(0, CScript([OP_TRUE]))) - tx4 = create_tx(tx1, 3, 0, CScript([OP_TRUE])) + tx4 = self.create_tx(tx1, 3, 0, CScript([OP_TRUE])) tx4.vout.append(CTxOut(0, CScript([OP_RETURN]))) - tx5 = create_tx(tx1, 4, 0, CScript([OP_RETURN])) + tx5 = self.create_tx(tx1, 4, 0, CScript([OP_RETURN])) - update_block(84, [tx1,tx2,tx3,tx4,tx5]) - yield accepted() - save_spendable_output() + b84 = self.update_block(84, [tx1, tx2, tx3, tx4, tx5]) + self.sync_blocks([b84], True) + self.save_spendable_output() - tip(83) - block(85, spend=out[29]) - yield rejected() + self.move_tip(83) + b85 = self.next_block(85, spend=out[29]) + self.sync_blocks([b85], False) # other chain is same length - block(86, spend=out[30]) - yield accepted() + b86 = self.next_block(86, spend=out[30]) + self.sync_blocks([b86], True) - tip(84) - block(87, spend=out[30]) - yield rejected() - save_spendable_output() + self.move_tip(84) + b87 = self.next_block(87, spend=out[30]) + self.sync_blocks([b87], False) # other chain is same length + self.save_spendable_output() - block(88, spend=out[31]) - yield accepted() - save_spendable_output() + b88 = self.next_block(88, spend=out[31]) + self.sync_blocks([b88], True) + self.save_spendable_output() # trying to spend the OP_RETURN output is rejected - block("89a", spend=out[32]) - tx = create_tx(tx1, 0, 0, CScript([OP_TRUE])) - update_block("89a", [tx]) - yield rejected() - - - # Test re-org of a week's worth of blocks (1088 blocks) - # This test takes a minute or two and can be accomplished in memory - # - if self.options.runbarelyexpensive: - tip(88) - LARGE_REORG_SIZE = 1088 - test1 = TestInstance(sync_every_block=False) - spend=out[32] - for i in range(89, LARGE_REORG_SIZE + 89): - b = block(i, spend) - tx = CTransaction() - script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 - script_output = CScript([b'\x00' * script_length]) - tx.vout.append(CTxOut(0, script_output)) - tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0))) - b = update_block(i, [tx]) - assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE) - test1.blocks_and_transactions.append([self.tip, True]) - save_spendable_output() - spend = get_spendable_output() - - yield test1 - chain1_tip = i - - # now create alt chain of same length - tip(88) - test2 = TestInstance(sync_every_block=False) - for i in range(89, LARGE_REORG_SIZE + 89): - block("alt"+str(i)) - test2.blocks_and_transactions.append([self.tip, False]) - yield test2 - - # extend alt chain to trigger re-org - block("alt" + str(chain1_tip + 1)) - yield accepted() - - # ... and re-org back to the first chain - tip(chain1_tip) - block(chain1_tip + 1) - yield rejected() - block(chain1_tip + 2) - yield accepted() - - chain1_tip += 2 + b89a = self.next_block("89a", spend=out[32]) + tx = self.create_tx(tx1, 0, 0, CScript([OP_TRUE])) + b89a = self.update_block("89a", [tx]) + self.sync_blocks([b89a], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True) + + self.log.info("Test a re-org of one week's worth of blocks (1088 blocks)") + + self.move_tip(88) + LARGE_REORG_SIZE = 1088 + blocks = [] + spend = out[32] + for i in range(89, LARGE_REORG_SIZE + 89): + b = self.next_block(i, spend) + tx = CTransaction() + script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 + script_output = CScript([b'\x00' * script_length]) + tx.vout.append(CTxOut(0, script_output)) + tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0))) + b = self.update_block(i, [tx]) + assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE) + blocks.append(b) + self.save_spendable_output() + spend = self.get_spendable_output() + + self.sync_blocks(blocks, True, timeout=180) + chain1_tip = i + + # now create alt chain of same length + self.move_tip(88) + blocks2 = [] + for i in range(89, LARGE_REORG_SIZE + 89): + blocks2.append(self.next_block("alt" + str(i))) + self.sync_blocks(blocks2, False, request_block=False) + + # extend alt chain to trigger re-org + block = self.next_block("alt" + str(chain1_tip + 1)) + self.sync_blocks([block], True, timeout=180) + + # ... and re-org back to the first chain + self.move_tip(chain1_tip) + block = self.next_block(chain1_tip + 1) + self.sync_blocks([block], False, request_block=False) + block = self.next_block(chain1_tip + 2) + self.sync_blocks([block], True, timeout=180) + + # Helper methods + ################ + + def add_transactions_to_block(self, block, tx_list): + [tx.rehash() for tx in tx_list] + block.vtx.extend(tx_list) + + # this is a little handier to use than the version in blocktools.py + def create_tx(self, spend_tx, n, value, script=CScript([OP_TRUE])): + return create_transaction(spend_tx, n, b"", value, script) + + # sign a transaction, using the key we know about + # this signs input 0 in tx, which is assumed to be spending output n in spend_tx + def sign_tx(self, tx, spend_tx, n): + scriptPubKey = bytearray(spend_tx.vout[n].scriptPubKey) + if (scriptPubKey[0] == OP_TRUE): # an anyone-can-spend + tx.vin[0].scriptSig = CScript() + return + (sighash, err) = SignatureHash(spend_tx.vout[n].scriptPubKey, tx, 0, SIGHASH_ALL) + tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))]) + + def create_and_sign_transaction(self, spend_tx, n, value, script=CScript([OP_TRUE])): + tx = self.create_tx(spend_tx, n, value, script) + self.sign_tx(tx, spend_tx, n) + tx.rehash() + return tx + + def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True): + if self.tip is None: + base_block_hash = self.genesis_hash + block_time = int(time.time()) + 1 + else: + base_block_hash = self.tip.sha256 + block_time = self.tip.nTime + 1 + # First create the coinbase + height = self.block_heights[base_block_hash] + 1 + coinbase = create_coinbase(height, self.coinbase_pubkey) + coinbase.vout[0].nValue += additional_coinbase_value + coinbase.rehash() + if spend is None: + block = create_block(base_block_hash, coinbase, block_time) + else: + coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 # all but one satoshi to fees + coinbase.rehash() + block = create_block(base_block_hash, coinbase, block_time) + tx = create_transaction(spend.tx, spend.n, b"", 1, script) # spend 1 satoshi + self.sign_tx(tx, spend.tx, spend.n) + self.add_transactions_to_block(block, [tx]) + block.hashMerkleRoot = block.calc_merkle_root() + if solve: + block.solve() + self.tip = block + self.block_heights[block.sha256] = height + assert number not in self.blocks + self.blocks[number] = block + return block + + # save the current tip so it can be spent by a later block + def save_spendable_output(self): + self.log.debug("saving spendable output %s" % self.tip.vtx[0]) + self.spendable_outputs.append(self.tip) + + # get an output that we previously marked as spendable + def get_spendable_output(self): + self.log.debug("getting spendable output %s" % self.spendable_outputs[0].vtx[0]) + return PreviousSpendableOutput(self.spendable_outputs.pop(0).vtx[0], 0) + + # move the tip back to a previous block + def move_tip(self, number): + self.tip = self.blocks[number] + + # adds transactions to the block and updates state + def update_block(self, block_number, new_transactions): + block = self.blocks[block_number] + self.add_transactions_to_block(block, new_transactions) + old_sha256 = block.sha256 + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + # Update the internal state just like in next_block + self.tip = block + if block.sha256 != old_sha256: + self.block_heights[block.sha256] = self.block_heights[old_sha256] + del self.block_heights[old_sha256] + self.blocks[block_number] = block + return block + + def reconnect_p2p(self): + """Add a P2P connection to the node. + + The node gets disconnected several times in this test. This helper + method reconnects the p2p and restarts the network thread.""" + + network_thread_join() + self.nodes[0].disconnect_p2ps() + self.nodes[0].add_p2p_connection(P2PDataStore()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() + + def sync_blocks(self, blocks, success=True, reject_code=None, reject_reason=None, request_block=True, reconnect=False, timeout=60): + """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. + Call with success = False if the tip shouldn't advance to the most recent block.""" + self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_code=reject_code, reject_reason=reject_reason, request_block=request_block, timeout=timeout) + if reconnect: + self.reconnect_p2p() if __name__ == '__main__': FullBlockTest().main() diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 8e97df2361..32a6bd5d59 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -133,12 +133,12 @@ class EstimateFeeTest(BitcoinTestFramework): which we will use to generate our transactions. """ self.add_nodes(3, extra_args=[["-maxorphantx=1000", "-whitelist=127.0.0.1"], - ["-maxorphantx=1000"], - ["-maxorphantx=1000"]]) + ["-blockmaxweight=68000", "-maxorphantx=1000"], + ["-blockmaxweight=32000", "-maxorphantx=1000"]]) # Use node0 to mine blocks for input splitting # Node1 mines small blocks but that are bigger than the expected transaction rate. - # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, - # (17k is room enough for 110 or so transactions) + # NOTE: the CreateNewBlock code starts counting block weight at 4,000 weight, + # (68k weight is room enough for 120 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 227ae6cb36..3adde8dd73 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -124,7 +124,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-checkblocks=5", "-disablesafemode"]) + self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5", "-disablesafemode"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) @@ -147,7 +147,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-checkblocks=5", "-disablesafemode"]) + self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5", "-disablesafemode"]) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 041e2b86e8..a24a2ec4f5 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -8,6 +8,7 @@ from base64 import b64encode from binascii import hexlify, unhexlify from decimal import Decimal, ROUND_DOWN import hashlib +import inspect import json import logging import os @@ -204,9 +205,9 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N if attempts == float('inf') and timeout == float('inf'): timeout = 60 attempt = 0 - timeout += time.time() + time_end = time.time() + timeout - while attempt < attempts and time.time() < timeout: + while attempt < attempts and time.time() < time_end: if lock: with lock: if predicate(): @@ -218,8 +219,12 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N time.sleep(0.05) # Print the cause of the timeout - assert_greater_than(attempts, attempt) - assert_greater_than(timeout, time.time()) + predicate_source = inspect.getsourcelines(predicate) + logger.error("wait_until() failed. Predicate: {}".format(predicate_source)) + if attempt >= attempts: + raise AssertionError("Predicate {} not true after {} attempts".format(predicate_source, attempts)) + elif time.time() >= time_end: + raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') # RPC/P2P connection constants and functions |