aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--configure.ac28
-rwxr-xr-xcontrib/devtools/clang-format-diff.py1
-rwxr-xr-xcontrib/devtools/security-check.py44
-rwxr-xr-xcontrib/devtools/symbol-check.py66
-rw-r--r--doc/developer-notes.md196
-rw-r--r--src/Makefile.am5
-rw-r--r--src/bitcoind.cpp10
-rw-r--r--src/dbwrapper.cpp31
-rw-r--r--src/init.cpp53
-rw-r--r--src/init.h4
-rw-r--r--src/qt/bitcoin.cpp7
-rw-r--r--src/rpc/blockchain.cpp6
-rw-r--r--src/rpc/rawtransaction.cpp12
-rw-r--r--src/validation.cpp1
-rw-r--r--src/validation.h1
-rw-r--r--src/wallet/init.cpp22
-rw-r--r--src/wallet/init.h45
-rw-r--r--src/walletinitinterface.h51
-rwxr-xr-xtest/functional/feature_block.py1308
-rwxr-xr-xtest/functional/feature_fee_estimation.py8
-rwxr-xr-xtest/functional/feature_pruning.py4
-rw-r--r--test/functional/test_framework/util.py13
22 files changed, 1102 insertions, 814 deletions
diff --git a/configure.ac b/configure.ac
index 4f57750a1a..c422914a26 100644
--- a/configure.ac
+++ b/configure.ac
@@ -219,6 +219,12 @@ AC_ARG_ENABLE([debug],
[enable_debug=$enableval],
[enable_debug=no])
+# Enable different -fsanitize options
+AC_ARG_WITH([sanitizers],
+ [AS_HELP_STRING([--with-sanitizers],
+ [comma separated list of extra sanitizers to build with (default is none enabled)])],
+ [use_sanitizers=$withval])
+
# Enable gprof profiling
AC_ARG_ENABLE([gprof],
[AS_HELP_STRING([--enable-gprof],
@@ -247,6 +253,26 @@ if test "x$enable_debug" = xyes; then
fi
fi
+if test x$use_sanitizers != x; then
+ # First check if the compiler accepts flags. If an incompatible pair like
+ # -fsanitize=address,thread is used here, this check will fail. This will also
+ # fail if a bad argument is passed, e.g. -fsanitize=undfeined
+ AX_CHECK_COMPILE_FLAG(
+ [[-fsanitize=$use_sanitizers]],
+ [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]],
+ [AC_MSG_ERROR([compiler did not accept requested flags])])
+
+ # Some compilers (e.g. GCC) require additional libraries like libasan,
+ # libtsan, libubsan, etc. Make sure linking still works with the sanitize
+ # flag. This is a separate check so we can give a better error message when
+ # the sanitize flags are supported by the compiler but the actual sanitizer
+ # libs are missing.
+ AX_CHECK_LINK_FLAG(
+ [[-fsanitize=$use_sanitizers]],
+ [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]],
+ [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])])
+fi
+
ERROR_CXXFLAGS=
if test "x$enable_werror" = "xyes"; then
if test "x$CXXFLAG_WERROR" = "x"; then
@@ -1258,6 +1284,8 @@ AC_SUBST(HARDENED_CPPFLAGS)
AC_SUBST(HARDENED_LDFLAGS)
AC_SUBST(PIC_FLAGS)
AC_SUBST(PIE_FLAGS)
+AC_SUBST(SANITIZER_CXXFLAGS)
+AC_SUBST(SANITIZER_LDFLAGS)
AC_SUBST(SSE42_CXXFLAGS)
AC_SUBST(LIBTOOL_APP_LDFLAGS)
AC_SUBST(USE_UPNP)
diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py
index ca1bd8854f..5402870fba 100755
--- a/contrib/devtools/clang-format-diff.py
+++ b/contrib/devtools/clang-format-diff.py
@@ -71,7 +71,6 @@ import argparse
import difflib
import io
import re
-import string
import subprocess
import sys
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
index b8f54192c9..0f2099953f 100755
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@@ -20,38 +20,38 @@ def check_ELF_PIE(executable):
'''
Check for position independent executable (PIE), allowing for address space randomization.
'''
- p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
ok = False
- for line in stdout.split(b'\n'):
+ for line in stdout.splitlines():
line = line.split()
- if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN':
+ if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN':
ok = True
return ok
def get_ELF_program_headers(executable):
'''Return type and flags for ELF program headers'''
- p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
in_headers = False
count = 0
headers = []
- for line in stdout.split(b'\n'):
- if line.startswith(b'Program Headers:'):
+ for line in stdout.splitlines():
+ if line.startswith('Program Headers:'):
in_headers = True
- if line == b'':
+ if line == '':
in_headers = False
if in_headers:
if count == 1: # header line
- ofs_typ = line.find(b'Type')
- ofs_offset = line.find(b'Offset')
- ofs_flags = line.find(b'Flg')
- ofs_align = line.find(b'Align')
+ ofs_typ = line.find('Type')
+ ofs_offset = line.find('Offset')
+ ofs_flags = line.find('Flg')
+ ofs_align = line.find('Align')
if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1:
raise ValueError('Cannot parse elfread -lW output')
elif count > 1:
@@ -68,9 +68,9 @@ def check_ELF_NX(executable):
have_wx = False
have_gnu_stack = False
for (typ, flags) in get_ELF_program_headers(executable):
- if typ == b'GNU_STACK':
+ if typ == 'GNU_STACK':
have_gnu_stack = True
- if b'W' in flags and b'E' in flags: # section is both writable and executable
+ if 'W' in flags and 'E' in flags: # section is both writable and executable
have_wx = True
return have_gnu_stack and not have_wx
@@ -87,17 +87,17 @@ def check_ELF_RELRO(executable):
# However, the dynamic linker need to write to this area so these are RW.
# Glibc itself takes care of mprotecting this area R after relocations are finished.
# See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347
- if typ == b'GNU_RELRO':
+ if typ == 'GNU_RELRO':
have_gnu_relro = True
have_bindnow = False
- p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
- for line in stdout.split(b'\n'):
+ for line in stdout.splitlines():
tokens = line.split()
- if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]):
+ if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]):
have_bindnow = True
return have_gnu_relro and have_bindnow
@@ -105,13 +105,13 @@ def check_ELF_Canary(executable):
'''
Check for use of stack canary
'''
- p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
ok = False
- for line in stdout.split(b'\n'):
- if b'__stack_chk_fail' in line:
+ for line in stdout.splitlines():
+ if '__stack_chk_fail' in line:
ok = True
return ok
@@ -121,13 +121,13 @@ def get_PE_dll_characteristics(executable):
Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386'
and bits is the DllCharacteristics value.
'''
- p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
arch = ''
bits = 0
- for line in stdout.split('\n'):
+ for line in stdout.splitlines():
tokens = line.split()
if len(tokens)>=2 and tokens[0] == 'architecture:':
arch = tokens[1].rstrip(',')
diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py
index 2b77857766..3a67319eaa 100755
--- a/contrib/devtools/symbol-check.py
+++ b/contrib/devtools/symbol-check.py
@@ -46,28 +46,28 @@ MAX_VERSIONS = {
# Ignore symbols that are exported as part of every executable
IGNORE_EXPORTS = {
-b'_edata', b'_end', b'_init', b'__bss_start', b'_fini', b'_IO_stdin_used'
+'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used'
}
READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt')
# Allowed NEEDED libraries
ALLOWED_LIBRARIES = {
# bitcoind and bitcoin-qt
-b'libgcc_s.so.1', # GCC base support
-b'libc.so.6', # C library
-b'libpthread.so.0', # threading
-b'libanl.so.1', # DNS resolve
-b'libm.so.6', # math library
-b'librt.so.1', # real-time (clock)
-b'ld-linux-x86-64.so.2', # 64-bit dynamic linker
-b'ld-linux.so.2', # 32-bit dynamic linker
+'libgcc_s.so.1', # GCC base support
+'libc.so.6', # C library
+'libpthread.so.0', # threading
+'libanl.so.1', # DNS resolve
+'libm.so.6', # math library
+'librt.so.1', # real-time (clock)
+'ld-linux-x86-64.so.2', # 64-bit dynamic linker
+'ld-linux.so.2', # 32-bit dynamic linker
# bitcoin-qt only
-b'libX11-xcb.so.1', # part of X11
-b'libX11.so.6', # part of X11
-b'libxcb.so.1', # part of X11
-b'libfontconfig.so.1', # font support
-b'libfreetype.so.6', # font parsing
-b'libdl.so.2' # programming interface to dynamic linker
+'libX11-xcb.so.1', # part of X11
+'libX11.so.6', # part of X11
+'libxcb.so.1', # part of X11
+'libfontconfig.so.1', # font support
+'libfreetype.so.6', # font parsing
+'libdl.so.2' # programming interface to dynamic linker
}
class CPPFilt(object):
@@ -77,10 +77,10 @@ class CPPFilt(object):
Use a pipe to the 'c++filt' command.
'''
def __init__(self):
- self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)
def __call__(self, mangled):
- self.proc.stdin.write(mangled + b'\n')
+ self.proc.stdin.write(mangled + '\n')
self.proc.stdin.flush()
return self.proc.stdout.readline().rstrip()
@@ -94,43 +94,43 @@ def read_symbols(executable, imports=True):
Parse an ELF executable and return a list of (symbol,version) tuples
for dynamic, imported symbols.
'''
- p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip()))
syms = []
- for line in stdout.split(b'\n'):
+ for line in stdout.splitlines():
line = line.split()
- if len(line)>7 and re.match(b'[0-9]+:$', line[0]):
- (sym, _, version) = line[7].partition(b'@')
- is_import = line[6] == b'UND'
- if version.startswith(b'@'):
+ if len(line)>7 and re.match('[0-9]+:$', line[0]):
+ (sym, _, version) = line[7].partition('@')
+ is_import = line[6] == 'UND'
+ if version.startswith('@'):
version = version[1:]
if is_import == imports:
syms.append((sym, version))
return syms
def check_version(max_versions, version):
- if b'_' in version:
- (lib, _, ver) = version.rpartition(b'_')
+ if '_' in version:
+ (lib, _, ver) = version.rpartition('_')
else:
lib = version
ver = '0'
- ver = tuple([int(x) for x in ver.split(b'.')])
+ ver = tuple([int(x) for x in ver.split('.')])
if not lib in max_versions:
return False
return ver <= max_versions[lib]
def read_libraries(filename):
- p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+ p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
raise IOError('Error opening file')
libraries = []
- for line in stdout.split(b'\n'):
+ for line in stdout.splitlines():
tokens = line.split()
- if len(tokens)>2 and tokens[1] == b'(NEEDED)':
- match = re.match(b'^Shared library: \[(.*)\]$', b' '.join(tokens[2:]))
+ if len(tokens)>2 and tokens[1] == '(NEEDED)':
+ match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
if match:
libraries.append(match.group(1))
else:
@@ -144,18 +144,18 @@ if __name__ == '__main__':
# Check imported symbols
for sym,version in read_symbols(filename, True):
if version and not check_version(MAX_VERSIONS, version):
- print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym).decode('utf-8'), version.decode('utf-8')))
+ print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version))
retval = 1
# Check exported symbols
for sym,version in read_symbols(filename, False):
if sym in IGNORE_EXPORTS:
continue
- print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym).decode('utf-8')))
+ print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym)))
retval = 1
# Check dependency libraries
for library_name in read_libraries(filename):
if library_name not in ALLOWED_LIBRARIES:
- print('%s: NEEDED library %s is not allowed' % (filename, library_name.decode('utf-8')))
+ print('%s: NEEDED library %s is not allowed' % (filename, library_name))
retval = 1
sys.exit(retval)
diff --git a/doc/developer-notes.md b/doc/developer-notes.md
index f10ad8e877..0de1892200 100644
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@@ -1,6 +1,43 @@
Developer Notes
===============
+<!-- markdown-toc start -->
+**Table of Contents**
+
+- [Developer Notes](#developer-notes)
+ - [Coding Style](#coding-style)
+ - [Doxygen comments](#doxygen-comments)
+ - [Development tips and tricks](#development-tips-and-tricks)
+ - [Compiling for debugging](#compiling-for-debugging)
+ - [Compiling for gprof profiling](#compiling-for-gprof-profiling)
+ - [debug.log](#debuglog)
+ - [Testnet and Regtest modes](#testnet-and-regtest-modes)
+ - [DEBUG_LOCKORDER](#debug_lockorder)
+ - [Valgrind suppressions file](#valgrind-suppressions-file)
+ - [Compiling for test coverage](#compiling-for-test-coverage)
+ - [Locking/mutex usage notes](#lockingmutex-usage-notes)
+ - [Threads](#threads)
+ - [Ignoring IDE/editor files](#ignoring-ideeditor-files)
+- [Development guidelines](#development-guidelines)
+ - [General Bitcoin Core](#general-bitcoin-core)
+ - [Wallet](#wallet)
+ - [General C++](#general-c)
+ - [C++ data structures](#c-data-structures)
+ - [Strings and formatting](#strings-and-formatting)
+ - [Variable names](#variable-names)
+ - [Threads and synchronization](#threads-and-synchronization)
+ - [Source code organization](#source-code-organization)
+ - [GUI](#gui)
+ - [Subtrees](#subtrees)
+ - [Git and GitHub tips](#git-and-github-tips)
+ - [Scripted diffs](#scripted-diffs)
+ - [RPC interface guidelines](#rpc-interface-guidelines)
+
+<!-- markdown-toc end -->
+
+Coding Style
+---------------
+
Various coding styles have been used during the history of the codebase,
and the result is not very consistent. However, we're now trying to converge to
a single style, which is specified below. When writing patches, favor the new
@@ -138,43 +175,44 @@ Documentation can be generated with `make docs` and cleaned up with `make clean-
Development tips and tricks
---------------------------
-**compiling for debugging**
+### Compiling for debugging
-Run configure with the --enable-debug option, then make. Or run configure with
-CXXFLAGS="-g -ggdb -O0" or whatever debug flags you need.
+Run configure with `--enable-debug` to add additional compiler flags that
+produce better debugging builds.
-**compiling for gprof profiling**
+### Compiling for gprof profiling
-Run configure with the --enable-gprof option, then make.
+Run configure with the `--enable-gprof` option, then make.
-**debug.log**
+### debug.log
If the code is behaving strangely, take a look in the debug.log file in the data directory;
error and debugging messages are written there.
-The -debug=... command-line option controls debugging; running with just -debug or -debug=1 will turn
+The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn
on all categories (and give you a very large debug.log file).
-The Qt code routes qDebug() output to debug.log under category "qt": run with -debug=qt
+The Qt code routes `qDebug()` output to debug.log under category "qt": run with `-debug=qt`
to see it.
-**testnet and regtest modes**
+### Testnet and Regtest modes
-Run with the -testnet option to run with "play bitcoins" on the test network, if you
+Run with the `-testnet` option to run with "play bitcoins" on the test network, if you
are testing multi-machine code that needs to operate across the internet.
-If you are testing something that can run on one machine, run with the -regtest option.
-In regression test mode, blocks can be created on-demand; see test/functional/ for tests
-that run in -regtest mode.
+If you are testing something that can run on one machine, run with the `-regtest` option.
+In regression test mode, blocks can be created on-demand; see [test/functional/](/test/functional) for tests
+that run in `-regtest` mode.
-**DEBUG_LOCKORDER**
+### DEBUG_LOCKORDER
-Bitcoin Core is a multithreaded application, and deadlocks or other multithreading bugs
-can be very difficult to track down. Compiling with -DDEBUG_LOCKORDER (configure
-CXXFLAGS="-DDEBUG_LOCKORDER -g") inserts run-time checks to keep track of which locks
-are held, and adds warnings to the debug.log file if inconsistencies are detected.
+Bitcoin Core is a multi-threaded application, and deadlocks or other
+multi-threading bugs can be very difficult to track down. The `--enable-debug`
+configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts
+run-time checks to keep track of which locks are held, and adds warnings to the
+debug.log file if inconsistencies are detected.
-**Valgrind suppressions file**
+### Valgrind suppressions file
Valgrind is a programming tool for memory debugging, memory leak detection, and
profiling. The repo contains a Valgrind suppressions file
@@ -189,7 +227,7 @@ $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \
$ valgrind -v --leak-check=full src/bitcoind -printtoconsole
```
-**compiling for test coverage**
+### Compiling for test coverage
LCOV can be used to generate a test coverage report based upon `make check`
execution. LCOV must be installed on your system (e.g. the `lcov` package
@@ -205,22 +243,73 @@ make cov
# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`.
```
+**Sanitizers**
+
+Bitcoin can be compiled with various "sanitizers" enabled, which add
+instrumentation for issues regarding things like memory safety, thread race
+conditions, or undefined behavior. This is controlled with the
+`--with-sanitizers` configure flag, which should be a comma separated list of
+sanitizers to enable. The sanitizer list should correspond to supported
+`-fsanitize=` options in your compiler. These sanitizers have runtime overhead,
+so they are most useful when testing changes or producing debugging builds.
+
+Some examples:
+
+```bash
+# Enable both the address sanitizer and the undefined behavior sanitizer
+./configure --with-sanitizers=address,undefined
+
+# Enable the thread sanitizer
+./configure --with-sanitizers=thread
+```
+
+If you are compiling with GCC you will typically need to install corresponding
+"san" libraries to actually compile with these flags, e.g. libasan for the
+address sanitizer, libtsan for the thread sanitizer, and libubsan for the
+undefined sanitizer. If you are missing required libraries, the configure script
+will fail with a linker error when testing the sanitizer flags.
+
+The test suite should pass cleanly with the `thread` and `undefined` sanitizers,
+but there are a number of known problems when using the `address` sanitizer. The
+address sanitizer is known to fail in
+[sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable
+unless you also use `--disable-asm` when running configure. We would like to fix
+sanitizer issues, so please send pull requests if you can fix any errors found
+by the address sanitizer (or any other sanitizer).
+
+Not all sanitizer options can be enabled at the same time, e.g. trying to build
+with `--with-sanitizers=address,thread` will fail in the configure script as
+these sanitizers are mutually incompatible. Refer to your compiler manual to
+learn more about these options and which sanitizers are supported by your
+compiler.
+
+Additional resources:
+
+ * [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html)
+ * [LeakSanitizer](https://clang.llvm.org/docs/LeakSanitizer.html)
+ * [MemorySanitizer](https://clang.llvm.org/docs/MemorySanitizer.html)
+ * [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html)
+ * [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html)
+ * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html)
+ * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki)
+ * [Issue #12691: Enable -fsanitize flags in Travis](https://github.com/bitcoin/bitcoin/issues/12691)
+
Locking/mutex usage notes
-------------------------
The code is multi-threaded, and uses mutexes and the
-LOCK/TRY_LOCK macros to protect data structures.
+`LOCK` and `TRY_LOCK` macros to protect data structures.
-Deadlocks due to inconsistent lock ordering (thread 1 locks cs_main
-and then cs_wallet, while thread 2 locks them in the opposite order:
-result, deadlock as each waits for the other to release its lock) are
-a problem. Compile with -DDEBUG_LOCKORDER to get lock order
-inconsistencies reported in the debug.log file.
+Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then
+`cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock
+as each waits for the other to release its lock) are a problem. Compile with
+`-DDEBUG_LOCKORDER` (or use `--enable-debug`) to get lock order inconsistencies
+reported in the debug.log file.
Re-architecting the core code so there are better-defined interfaces
between the various components is a goal, with any necessary locking
-done by the components (e.g. see the self-contained CBasicKeyStore class
-and its cs_KeyStore lock for example).
+done by the components (e.g. see the self-contained `CBasicKeyStore` class
+and its `cs_KeyStore` lock for example).
Threads
-------
@@ -543,7 +632,10 @@ its upstream repository.
Current subtrees include:
- src/leveldb
- - Upstream at https://github.com/google/leveldb ; Maintained by Google, but open important PRs to Core to avoid delay
+ - Upstream at https://github.com/google/leveldb ; Maintained by Google, but
+ open important PRs to Core to avoid delay.
+ - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when
+ merging upstream changes to the leveldb subtree.
- src/libsecp256k1
- Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintaned by Core contributors.
@@ -554,6 +646,52 @@ Current subtrees include:
- src/univalue
- Upstream at https://github.com/jgarzik/univalue ; report important PRs to Core to avoid delay.
+Upgrading LevelDB
+---------------------
+
+Extra care must be taken when upgrading LevelDB. This section explains issues
+you must be aware of.
+
+### File Descriptor Counts
+
+In most configurations we use the default LevelDB value for `max_open_files`,
+which is 1000 at the time of this writing. If LevelDB actually uses this many
+file descriptors it will cause problems with Bitcoin's `select()` loop, because
+it may cause new sockets to be created where the fd value is >= 1024. For this
+reason, on 64-bit Unix systems we rely on an internal LevelDB optimization that
+uses `mmap()` + `close()` to open table files without actually retaining
+references to the table file descriptors. If you are upgrading LevelDB, you must
+sanity check the changes to make sure that this assumption remains valid.
+
+In addition to reviewing the upstream changes in `env_posix.cc`, you can use `lsof` to
+check this. For example, on Linux this command will show open `.ldb` file counts:
+
+```bash
+$ lsof -p $(pidof bitcoind) |\
+ awk 'BEGIN { fd=0; mem=0; } /ldb$/ { if ($4 == "mem") mem++; else fd++ } END { printf "mem = %s, fd = %s\n", mem, fd}'
+mem = 119, fd = 0
+```
+
+The `mem` value shows how many files are mmap'ed, and the `fd` value shows you
+many file descriptors these files are using. You should check that `fd` is a
+small number (usually 0 on 64-bit hosts).
+
+See the notes in the `SetMaxOpenFiles()` function in `dbwrapper.cc` for more
+details.
+
+### Consensus Compatibility
+
+It is possible for LevelDB changes to inadvertently change consensus
+compatibility between nodes. This happened in Bitcoin 0.8 (when LevelDB was
+first introduced). When upgrading LevelDB you should review the upstream changes
+to check for issues affecting consensus compatibility.
+
+For example, if LevelDB had a bug that accidentally prevented a key from being
+returned in an edge case, and that bug was fixed upstream, the bug "fix" would
+be an incompatible consensus change. In this situation the correct behavior
+would be to revert the upstream fix before applying the updates to Bitcoin's
+copy of LevelDB. In general you should be wary of any upstream changes affecting
+what data is returned from LevelDB queries.
Git and GitHub tips
---------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 7c2fe56d9d..72e5cdb95d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -4,8 +4,8 @@
DIST_SUBDIRS = secp256k1 univalue
-AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS)
-AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS)
+AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS)
+AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS)
AM_CPPFLAGS = $(HARDENED_CPPFLAGS)
EXTRA_LIBRARIES =
@@ -162,6 +162,7 @@ BITCOIN_CORE_H = \
validation.h \
validationinterface.h \
versionbits.h \
+ walletinitinterface.h \
wallet/coincontrol.h \
wallet/crypter.h \
wallet/db.h \
diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp
index db5e610684..df7fad89c2 100644
--- a/src/bitcoind.cpp
+++ b/src/bitcoind.cpp
@@ -18,6 +18,10 @@
#include <httpserver.h>
#include <httprpc.h>
#include <utilstrencodings.h>
+#if ENABLE_WALLET
+#include <wallet/init.h>
+#endif
+#include <walletinitinterface.h>
#include <boost/thread.hpp>
@@ -59,6 +63,12 @@ bool AppInit(int argc, char* argv[])
{
bool fRet = false;
+#if ENABLE_WALLET
+ g_wallet_init_interface.reset(new WalletInit);
+#else
+ g_wallet_init_interface.reset(new DummyWalletInit);
+#endif
+
//
// Parameters
//
diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp
index fb0d4215a2..752f985bc0 100644
--- a/src/dbwrapper.cpp
+++ b/src/dbwrapper.cpp
@@ -71,6 +71,31 @@ public:
}
};
+static void SetMaxOpenFiles(leveldb::Options *options) {
+ // On most platforms the default setting of max_open_files (which is 1000)
+ // is optimal. On Windows using a large file count is OK because the handles
+ // do not interfere with select() loops. On 64-bit Unix hosts this value is
+ // also OK, because up to that amount LevelDB will use an mmap
+ // implementation that does not use extra file descriptors (the fds are
+ // closed after being mmaped).
+ //
+ // Increasing the value beyond the default is dangerous because LevelDB will
+ // fall back to a non-mmap implementation when the file count is too large.
+ // On 32-bit Unix host we should decrease the value because the handles use
+ // up real fds, and we want to avoid fd exhaustion issues.
+ //
+ // See PR #12495 for further discussion.
+
+ int default_open_files = options->max_open_files;
+#ifndef WIN32
+ if (sizeof(void*) < 8) {
+ options->max_open_files = 64;
+ }
+#endif
+ LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
+ options->max_open_files, default_open_files);
+}
+
static leveldb::Options GetOptions(size_t nCacheSize)
{
leveldb::Options options;
@@ -78,13 +103,13 @@ static leveldb::Options GetOptions(size_t nCacheSize)
options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
options.compression = leveldb::kNoCompression;
- options.max_open_files = 64;
options.info_log = new CBitcoinLevelDBLogger();
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
// LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
// on corruption in later versions.
options.paranoid_checks = true;
}
+ SetMaxOpenFiles(&options);
return options;
}
@@ -159,12 +184,12 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
double mem_before = 0;
if (log_memory) {
- mem_before = DynamicMemoryUsage() / 1024 / 1024;
+ mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
}
leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
dbwrapper_private::HandleError(status);
if (log_memory) {
- double mem_after = DynamicMemoryUsage() / 1024 / 1024;
+ double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
m_name, mem_before, mem_after);
}
diff --git a/src/init.cpp b/src/init.cpp
index b0ba0cb835..3eb2a83243 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -43,10 +43,8 @@
#include <util.h>
#include <utilmoneystr.h>
#include <validationinterface.h>
-#ifdef ENABLE_WALLET
-#include <wallet/init.h>
-#endif
#include <warnings.h>
+#include <walletinitinterface.h>
#include <stdint.h>
#include <stdio.h>
#include <memory>
@@ -74,6 +72,7 @@ static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false;
std::unique_ptr<CConnman> g_connman;
std::unique_ptr<PeerLogicValidation> peerLogic;
+std::unique_ptr<WalletInitInterface> g_wallet_init_interface;
#if ENABLE_ZMQ
static CZMQNotificationInterface* pzmqNotificationInterface = nullptr;
@@ -116,7 +115,6 @@ static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat";
//
std::atomic<bool> fRequestShutdown(false);
-std::atomic<bool> fDumpMempoolLater(false);
void StartShutdown()
{
@@ -189,9 +187,7 @@ void Shutdown()
StopREST();
StopRPC();
StopHTTPServer();
-#ifdef ENABLE_WALLET
- FlushWallets();
-#endif
+ g_wallet_init_interface->Flush();
StopMapPort();
// Because these depend on each-other, we make sure that neither can be
@@ -208,7 +204,7 @@ void Shutdown()
threadGroup.interrupt_all();
threadGroup.join_all();
- if (fDumpMempoolLater && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
+ if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
DumpMempool();
}
@@ -249,9 +245,7 @@ void Shutdown()
pcoinsdbview.reset();
pblocktree.reset();
}
-#ifdef ENABLE_WALLET
- StopWallets();
-#endif
+ g_wallet_init_interface->Stop();
#if ENABLE_ZMQ
if (pzmqNotificationInterface) {
@@ -271,9 +265,8 @@ void Shutdown()
UnregisterAllValidationInterfaces();
GetMainSignals().UnregisterBackgroundSignalScheduler();
GetMainSignals().UnregisterWithMempoolSignals(mempool);
-#ifdef ENABLE_WALLET
- CloseWallets();
-#endif
+ g_wallet_init_interface->Close();
+ g_wallet_init_interface.reset();
globalVerifyHandle.reset();
ECC_Stop();
LogPrintf("%s: done\n", __func__);
@@ -416,9 +409,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-whitelist=<IP address or network>", _("Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times.") +
" " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway"));
-#ifdef ENABLE_WALLET
- strUsage += GetWalletHelpString(showDebug);
-#endif
+ strUsage += g_wallet_init_interface->GetHelpString(showDebug);
#if ENABLE_ZMQ
strUsage += HelpMessageGroup(_("ZeroMQ notification options:"));
@@ -692,8 +683,8 @@ void ThreadImport(std::vector<fs::path> vImportFiles)
} // End scope of CImportingNow
if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
LoadMempool();
- fDumpMempoolLater = !fRequestShutdown;
}
+ g_is_mempool_loaded = !fRequestShutdown;
}
/** Sanity checks
@@ -1087,10 +1078,7 @@ bool AppInitParameterInteraction()
return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString()));
nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp);
-#ifdef ENABLE_WALLET
- if (!WalletParameterInteraction())
- return false;
-#endif
+ if (!g_wallet_init_interface->ParameterInteraction()) return false;
fIsBareMultisigStd = gArgs.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG);
fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER);
@@ -1256,9 +1244,7 @@ bool AppInitMain()
* available in the GUI RPC console even if external calls are disabled.
*/
RegisterAllCoreRPCCommands(tableRPC);
-#ifdef ENABLE_WALLET
- RegisterWalletRPC(tableRPC);
-#endif
+ g_wallet_init_interface->RegisterRPC(tableRPC);
/* Start the RPC server already. It will be started in "warmup" mode
* and not really process calls already (but it will signify connections
@@ -1275,10 +1261,8 @@ bool AppInitMain()
int64_t nStart;
// ********************************************************* Step 5: verify wallet database integrity
-#ifdef ENABLE_WALLET
- if (!VerifyWallets())
- return false;
-#endif
+ if (!g_wallet_init_interface->Verify()) return false;
+
// ********************************************************* Step 6: network initialization
// Note that we absolutely cannot open any actual connections
// until the very end ("start node") as the UTXO/block state
@@ -1596,12 +1580,7 @@ bool AppInitMain()
fFeeEstimatesInitialized = true;
// ********************************************************* Step 8: load wallet
-#ifdef ENABLE_WALLET
- if (!OpenWallets())
- return false;
-#else
- LogPrintf("No wallet support compiled in!\n");
-#endif
+ if (!g_wallet_init_interface->Open()) return false;
// ********************************************************* Step 9: data directory maintenance
@@ -1747,9 +1726,7 @@ bool AppInitMain()
SetRPCWarmupFinished();
uiInterface.InitMessage(_("Done loading"));
-#ifdef ENABLE_WALLET
- StartWallets(scheduler);
-#endif
+ g_wallet_init_interface->Start(scheduler);
return true;
}
diff --git a/src/init.h b/src/init.h
index 6f75a43e62..c93a210154 100644
--- a/src/init.h
+++ b/src/init.h
@@ -6,11 +6,15 @@
#ifndef BITCOIN_INIT_H
#define BITCOIN_INIT_H
+#include <memory>
#include <string>
class CScheduler;
class CWallet;
+class WalletInitInterface;
+extern std::unique_ptr<WalletInitInterface> g_wallet_init_interface;
+
namespace boost
{
class thread_group;
diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp
index f1298df8ca..f853c04617 100644
--- a/src/qt/bitcoin.cpp
+++ b/src/qt/bitcoin.cpp
@@ -34,8 +34,10 @@
#include <warnings.h>
#ifdef ENABLE_WALLET
+#include <wallet/init.h>
#include <wallet/wallet.h>
#endif
+#include <walletinitinterface.h>
#include <stdint.h>
@@ -677,6 +679,11 @@ int main(int argc, char *argv[])
// Start up the payment server early, too, so impatient users that click on
// bitcoin: links repeatedly have their payment requests routed to this process:
app.createPaymentServer();
+
+ // Hook up the wallet init interface
+ g_wallet_init_interface.reset(new WalletInit);
+#else
+ g_wallet_init_interface.reset(new DummyWalletInit);
#endif
/// 9. Main GUI initialization
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index e15dad2f0b..31cbec4c86 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1607,13 +1607,17 @@ UniValue savemempool(const JSONRPCRequest& request)
if (request.fHelp || request.params.size() != 0) {
throw std::runtime_error(
"savemempool\n"
- "\nDumps the mempool to disk.\n"
+ "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n"
"\nExamples:\n"
+ HelpExampleCli("savemempool", "")
+ HelpExampleRpc("savemempool", "")
);
}
+ if (!g_is_mempool_loaded) {
+ throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet");
+ }
+
if (!DumpMempool()) {
throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk");
}
diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp
index 20bfd3f355..77040f75fd 100644
--- a/src/rpc/rawtransaction.cpp
+++ b/src/rpc/rawtransaction.cpp
@@ -1023,18 +1023,18 @@ UniValue signrawtransaction(const JSONRPCRequest& request)
new_request.params.push_back(request.params[1]);
new_request.params.push_back(request.params[3]);
return signrawtransactionwithkey(new_request);
- }
- // Otherwise sign with the wallet which does not take a privkeys parameter
+ } else {
#ifdef ENABLE_WALLET
- else {
+ // Otherwise sign with the wallet which does not take a privkeys parameter
new_request.params.push_back(request.params[0]);
new_request.params.push_back(request.params[1]);
new_request.params.push_back(request.params[3]);
return signrawtransactionwithwallet(new_request);
- }
+#else
+ // If we have made it this far, then wallet is disabled and no private keys were given, so fail here.
+ throw JSONRPCError(RPC_INVALID_PARAMETER, "No private keys available.");
#endif
- // If we have made it this far, then wallet is disabled and no private keys were given, so fail here.
- throw JSONRPCError(RPC_INVALID_PARAMETER, "No private keys available.");
+ }
}
UniValue sendrawtransaction(const JSONRPCRequest& request)
diff --git a/src/validation.cpp b/src/validation.cpp
index 839b248143..d6848a3aca 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -227,6 +227,7 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE;
CBlockPolicyEstimator feeEstimator;
CTxMemPool mempool(&feeEstimator);
+std::atomic_bool g_is_mempool_loaded{false};
/** Constant stuff for coinbase transactions we create: */
CScript COINBASE_FLAGS;
diff --git a/src/validation.h b/src/validation.h
index 0a78075316..4031989f00 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -158,6 +158,7 @@ extern CScript COINBASE_FLAGS;
extern CCriticalSection cs_main;
extern CBlockPolicyEstimator feeEstimator;
extern CTxMemPool mempool;
+extern std::atomic_bool g_is_mempool_loaded;
typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap;
extern BlockMap& mapBlockIndex;
extern uint64_t nLastBlockTx;
diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp
index 61481e01b6..3d7bb674f0 100644
--- a/src/wallet/init.cpp
+++ b/src/wallet/init.cpp
@@ -14,7 +14,7 @@
#include <wallet/wallet.h>
#include <wallet/walletutil.h>
-std::string GetWalletHelpString(bool showDebug)
+std::string WalletInit::GetHelpString(bool showDebug)
{
std::string strUsage = HelpMessageGroup(_("Wallet options:"));
strUsage += HelpMessageOpt("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)));
@@ -56,7 +56,7 @@ std::string GetWalletHelpString(bool showDebug)
return strUsage;
}
-bool WalletParameterInteraction()
+bool WalletInit::ParameterInteraction()
{
if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
for (const std::string& wallet : gArgs.GetArgs("-wallet")) {
@@ -184,7 +184,7 @@ bool WalletParameterInteraction()
return true;
}
-void RegisterWalletRPC(CRPCTable &t)
+void WalletInit::RegisterRPC(CRPCTable &t)
{
if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
return;
@@ -193,7 +193,7 @@ void RegisterWalletRPC(CRPCTable &t)
RegisterWalletRPCCommands(t);
}
-bool VerifyWallets()
+bool WalletInit::Verify()
{
if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
return true;
@@ -268,7 +268,7 @@ bool VerifyWallets()
return true;
}
-bool OpenWallets()
+bool WalletInit::Open()
{
if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) {
LogPrintf("Wallet disabled!\n");
@@ -286,25 +286,29 @@ bool OpenWallets()
return true;
}
-void StartWallets(CScheduler& scheduler) {
+void WalletInit::Start(CScheduler& scheduler)
+{
for (CWalletRef pwallet : vpwallets) {
pwallet->postInitProcess(scheduler);
}
}
-void FlushWallets() {
+void WalletInit::Flush()
+{
for (CWalletRef pwallet : vpwallets) {
pwallet->Flush(false);
}
}
-void StopWallets() {
+void WalletInit::Stop()
+{
for (CWalletRef pwallet : vpwallets) {
pwallet->Flush(true);
}
}
-void CloseWallets() {
+void WalletInit::Close()
+{
for (CWalletRef pwallet : vpwallets) {
delete pwallet;
}
diff --git a/src/wallet/init.h b/src/wallet/init.h
index 0b3ee2dda2..f8be90d3e3 100644
--- a/src/wallet/init.h
+++ b/src/wallet/init.h
@@ -6,38 +6,43 @@
#ifndef BITCOIN_WALLET_INIT_H
#define BITCOIN_WALLET_INIT_H
+#include <walletinitinterface.h>
#include <string>
class CRPCTable;
class CScheduler;
-//! Return the wallets help message.
-std::string GetWalletHelpString(bool showDebug);
+class WalletInit : public WalletInitInterface {
+public:
-//! Wallets parameter interaction
-bool WalletParameterInteraction();
+ //! Return the wallets help message.
+ std::string GetHelpString(bool showDebug) override;
-//! Register wallet RPCs.
-void RegisterWalletRPC(CRPCTable &tableRPC);
+ //! Wallets parameter interaction
+ bool ParameterInteraction() override;
-//! Responsible for reading and validating the -wallet arguments and verifying the wallet database.
-// This function will perform salvage on the wallet if requested, as long as only one wallet is
-// being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet).
-bool VerifyWallets();
+ //! Register wallet RPCs.
+ void RegisterRPC(CRPCTable &tableRPC) override;
-//! Load wallet databases.
-bool OpenWallets();
+ //! Responsible for reading and validating the -wallet arguments and verifying the wallet database.
+ // This function will perform salvage on the wallet if requested, as long as only one wallet is
+ // being loaded (WalletParameterInteraction forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet).
+ bool Verify() override;
-//! Complete startup of wallets.
-void StartWallets(CScheduler& scheduler);
+ //! Load wallet databases.
+ bool Open() override;
-//! Flush all wallets in preparation for shutdown.
-void FlushWallets();
+ //! Complete startup of wallets.
+ void Start(CScheduler& scheduler) override;
-//! Stop all wallets. Wallets will be flushed first.
-void StopWallets();
+ //! Flush all wallets in preparation for shutdown.
+ void Flush() override;
-//! Close all wallets.
-void CloseWallets();
+ //! Stop all wallets. Wallets will be flushed first.
+ void Stop() override;
+
+ //! Close all wallets.
+ void Close() override;
+};
#endif // BITCOIN_WALLET_INIT_H
diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h
new file mode 100644
index 0000000000..47e4e2cce1
--- /dev/null
+++ b/src/walletinitinterface.h
@@ -0,0 +1,51 @@
+// Copyright (c) 2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef WALLETINITINTERFACE_H
+#define WALLETINITINTERFACE_H
+
+#include <string>
+
+class CScheduler;
+class CRPCTable;
+
+class WalletInitInterface {
+public:
+ /** Get wallet help string */
+ virtual std::string GetHelpString(bool showDebug) = 0;
+ /** Check wallet parameter interaction */
+ virtual bool ParameterInteraction() = 0;
+ /** Register wallet RPC*/
+ virtual void RegisterRPC(CRPCTable &) = 0;
+ /** Verify wallets */
+ virtual bool Verify() = 0;
+ /** Open wallets*/
+ virtual bool Open() = 0;
+ /** Start wallets*/
+ virtual void Start(CScheduler& scheduler) = 0;
+ /** Flush Wallets*/
+ virtual void Flush() = 0;
+ /** Stop Wallets*/
+ virtual void Stop() = 0;
+ /** Close wallets */
+ virtual void Close() = 0;
+
+ virtual ~WalletInitInterface() {}
+};
+
+class DummyWalletInit : public WalletInitInterface {
+public:
+
+ std::string GetHelpString(bool showDebug) override {return std::string{};}
+ bool ParameterInteraction() override {return true;}
+ void RegisterRPC(CRPCTable &) override {}
+ bool Verify() override {return true;}
+ bool Open() override {return true;}
+ void Start(CScheduler& scheduler) override {}
+ void Flush() override {}
+ void Stop() override {}
+ void Close() override {}
+};
+
+#endif // WALLETINITINTERFACE_H
diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py
index fe9bbda14b..181c7f3369 100755
--- a/test/functional/feature_block.py
+++ b/test/functional/feature_block.py
@@ -2,36 +2,59 @@
# Copyright (c) 2015-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test block processing.
-
-This reimplements tests from the bitcoinj/FullBlockTestGenerator used
-by the pull-tester.
-
-We use the testing framework in which we expect a particular answer from
-each test.
-"""
-
-from test_framework.test_framework import ComparisonTestFramework
-from test_framework.util import *
-from test_framework.comptool import TestManager, TestInstance, RejectResult
-from test_framework.blocktools import *
+"""Test block processing."""
+import copy
+import struct
import time
+
+from test_framework.blocktools import create_block, create_coinbase, create_transaction, get_legacy_sigopcount_block
from test_framework.key import CECKey
-from test_framework.script import *
-from test_framework.mininode import network_thread_start
-import struct
+from test_framework.messages import (
+ CBlock,
+ COIN,
+ COutPoint,
+ CTransaction,
+ CTxIn,
+ CTxOut,
+ MAX_BLOCK_BASE_SIZE,
+ uint256_from_compact,
+ uint256_from_str,
+)
+from test_framework.mininode import P2PDataStore, network_thread_start, network_thread_join
+from test_framework.script import (
+ CScript,
+ MAX_SCRIPT_ELEMENT_SIZE,
+ OP_2DUP,
+ OP_CHECKMULTISIG,
+ OP_CHECKMULTISIGVERIFY,
+ OP_CHECKSIG,
+ OP_CHECKSIGVERIFY,
+ OP_ELSE,
+ OP_ENDIF,
+ OP_EQUAL,
+ OP_FALSE,
+ OP_HASH160,
+ OP_IF,
+ OP_INVALIDOPCODE,
+ OP_RETURN,
+ OP_TRUE,
+ SIGHASH_ALL,
+ SignatureHash,
+ hash160,
+)
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+MAX_BLOCK_SIGOPS = 20000
class PreviousSpendableOutput():
- def __init__(self, tx = CTransaction(), n = -1):
+ def __init__(self, tx=CTransaction(), n=-1):
self.tx = tx
self.n = n # the output we're spending
# Use this class for tests that require behavior other than normal "mininode" behavior.
# For now, it is used to serialize a bloated varint (b64).
class CBrokenBlock(CBlock):
- def __init__(self, header=None):
- super(CBrokenBlock, self).__init__(header)
-
def initialize(self, base_block):
self.vtx = copy.deepcopy(base_block.vtx)
self.hashMerkleRoot = self.calc_merkle_root()
@@ -48,381 +71,272 @@ class CBrokenBlock(CBlock):
return r
def normal_serialize(self):
- r = b""
- r += super(CBrokenBlock, self).serialize()
- return r
+ return super().serialize()
-class FullBlockTest(ComparisonTestFramework):
- # Can either run this test as 1 node with expected answers, or two and compare them.
- # Change the "outcome" variable from each TestInstance object to only do the comparison.
+class FullBlockTest(BitcoinTestFramework):
def set_test_params(self):
self.num_nodes = 1
self.setup_clean_chain = True
+ self.extra_args = [[]]
+
+ def run_test(self):
+ node = self.nodes[0] # convenience reference to the node
+
+ # reconnect_p2p() expects the network thread to be running
+ network_thread_start()
+
+ self.reconnect_p2p()
+
self.block_heights = {}
self.coinbase_key = CECKey()
self.coinbase_key.set_secretbytes(b"horsebattery")
self.coinbase_pubkey = self.coinbase_key.get_pubkey()
self.tip = None
self.blocks = {}
-
- def add_options(self, parser):
- super().add_options(parser)
- parser.add_option("--runbarelyexpensive", dest="runbarelyexpensive", default=True)
-
- def run_test(self):
- self.test = TestManager(self, self.options.tmpdir)
- self.test.add_all_connections(self.nodes)
- network_thread_start()
- self.test.run()
-
- def add_transactions_to_block(self, block, tx_list):
- [ tx.rehash() for tx in tx_list ]
- block.vtx.extend(tx_list)
-
- # this is a little handier to use than the version in blocktools.py
- def create_tx(self, spend_tx, n, value, script=CScript([OP_TRUE])):
- tx = create_transaction(spend_tx, n, b"", value, script)
- return tx
-
- # sign a transaction, using the key we know about
- # this signs input 0 in tx, which is assumed to be spending output n in spend_tx
- def sign_tx(self, tx, spend_tx, n):
- scriptPubKey = bytearray(spend_tx.vout[n].scriptPubKey)
- if (scriptPubKey[0] == OP_TRUE): # an anyone-can-spend
- tx.vin[0].scriptSig = CScript()
- return
- (sighash, err) = SignatureHash(spend_tx.vout[n].scriptPubKey, tx, 0, SIGHASH_ALL)
- tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))])
-
- def create_and_sign_transaction(self, spend_tx, n, value, script=CScript([OP_TRUE])):
- tx = self.create_tx(spend_tx, n, value, script)
- self.sign_tx(tx, spend_tx, n)
- tx.rehash()
- return tx
-
- def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True):
- if self.tip == None:
- base_block_hash = self.genesis_hash
- block_time = int(time.time())+1
- else:
- base_block_hash = self.tip.sha256
- block_time = self.tip.nTime + 1
- # First create the coinbase
- height = self.block_heights[base_block_hash] + 1
- coinbase = create_coinbase(height, self.coinbase_pubkey)
- coinbase.vout[0].nValue += additional_coinbase_value
- coinbase.rehash()
- if spend == None:
- block = create_block(base_block_hash, coinbase, block_time)
- else:
- coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 # all but one satoshi to fees
- coinbase.rehash()
- block = create_block(base_block_hash, coinbase, block_time)
- tx = create_transaction(spend.tx, spend.n, b"", 1, script) # spend 1 satoshi
- self.sign_tx(tx, spend.tx, spend.n)
- self.add_transactions_to_block(block, [tx])
- block.hashMerkleRoot = block.calc_merkle_root()
- if solve:
- block.solve()
- self.tip = block
- self.block_heights[block.sha256] = height
- assert number not in self.blocks
- self.blocks[number] = block
- return block
-
- def get_tests(self):
self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16)
self.block_heights[self.genesis_hash] = 0
- spendable_outputs = []
-
- # save the current tip so it can be spent by a later block
- def save_spendable_output():
- spendable_outputs.append(self.tip)
-
- # get an output that we previously marked as spendable
- def get_spendable_output():
- return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0)
-
- # returns a test case that asserts that the current tip was accepted
- def accepted():
- return TestInstance([[self.tip, True]])
-
- # returns a test case that asserts that the current tip was rejected
- def rejected(reject = None):
- if reject is None:
- return TestInstance([[self.tip, False]])
- else:
- return TestInstance([[self.tip, reject]])
-
- # move the tip back to a previous block
- def tip(number):
- self.tip = self.blocks[number]
-
- # adds transactions to the block and updates state
- def update_block(block_number, new_transactions):
- block = self.blocks[block_number]
- self.add_transactions_to_block(block, new_transactions)
- old_sha256 = block.sha256
- block.hashMerkleRoot = block.calc_merkle_root()
- block.solve()
- # Update the internal state just like in next_block
- self.tip = block
- if block.sha256 != old_sha256:
- self.block_heights[block.sha256] = self.block_heights[old_sha256]
- del self.block_heights[old_sha256]
- self.blocks[block_number] = block
- return block
-
- # shorthand for functions
- block = self.next_block
- create_tx = self.create_tx
- create_and_sign_tx = self.create_and_sign_transaction
-
- # these must be updated if consensus changes
- MAX_BLOCK_SIGOPS = 20000
-
+ self.spendable_outputs = []
# Create a new block
- block(0)
- save_spendable_output()
- yield accepted()
-
+ b0 = self.next_block(0)
+ self.save_spendable_output()
+ self.sync_blocks([b0])
- # Now we need that block to mature so we can spend the coinbase.
- test = TestInstance(sync_every_block=False)
+ # Allow the block to mature
+ blocks = []
for i in range(99):
- block(5000 + i)
- test.blocks_and_transactions.append([self.tip, True])
- save_spendable_output()
- yield test
+ blocks.append(self.next_block(5000 + i))
+ self.save_spendable_output()
+ self.sync_blocks(blocks)
# collect spendable outputs now to avoid cluttering the code later on
out = []
for i in range(33):
- out.append(get_spendable_output())
+ out.append(self.get_spendable_output())
# Start by building a couple of blocks on top (which output is spent is
# in parentheses):
# genesis -> b1 (0) -> b2 (1)
- block(1, spend=out[0])
- save_spendable_output()
- yield accepted()
+ b1 = self.next_block(1, spend=out[0])
+ self.save_spendable_output()
- block(2, spend=out[1])
- yield accepted()
- save_spendable_output()
+ b2 = self.next_block(2, spend=out[1])
+ self.save_spendable_output()
- # so fork like this:
+ self.sync_blocks([b1, b2])
+
+ # Fork like this:
#
# genesis -> b1 (0) -> b2 (1)
# \-> b3 (1)
#
# Nothing should happen at this point. We saw b2 first so it takes priority.
- tip(1)
- b3 = block(3, spend=out[1])
+ self.log.info("Don't reorg to a chain of the same length")
+ self.move_tip(1)
+ b3 = self.next_block(3, spend=out[1])
txout_b3 = PreviousSpendableOutput(b3.vtx[1], 0)
- yield rejected()
-
+ self.sync_blocks([b3], False)
# Now we add another block to make the alternative chain longer.
#
# genesis -> b1 (0) -> b2 (1)
# \-> b3 (1) -> b4 (2)
- block(4, spend=out[2])
- yield accepted()
-
+ self.log.info("Reorg to a longer chain")
+ b4 = self.next_block(4, spend=out[2])
+ self.sync_blocks([b4])
# ... and back to the first chain.
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b3 (1) -> b4 (2)
- tip(2)
- block(5, spend=out[2])
- save_spendable_output()
- yield rejected()
+ self.move_tip(2)
+ b5 = self.next_block(5, spend=out[2])
+ self.save_spendable_output()
+ self.sync_blocks([b5], False)
- block(6, spend=out[3])
- yield accepted()
+ self.log.info("Reorg back to the original chain")
+ b6 = self.next_block(6, spend=out[3])
+ self.sync_blocks([b6], True)
# Try to create a fork that double-spends
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b7 (2) -> b8 (4)
# \-> b3 (1) -> b4 (2)
- tip(5)
- block(7, spend=out[2])
- yield rejected()
+ self.log.info("Reject a chain with a double spend, even if it is longer")
+ self.move_tip(5)
+ b7 = self.next_block(7, spend=out[2])
+ self.sync_blocks([b7], False)
- block(8, spend=out[4])
- yield rejected()
+ b8 = self.next_block(8, spend=out[4])
+ self.sync_blocks([b8], False, reconnect=True)
# Try to create a block that has too much fee
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b9 (4)
# \-> b3 (1) -> b4 (2)
- tip(6)
- block(9, spend=out[4], additional_coinbase_value=1)
- yield rejected(RejectResult(16, b'bad-cb-amount'))
+ self.log.info("Reject a block where the miner creates too much coinbase reward")
+ self.move_tip(6)
+ b9 = self.next_block(9, spend=out[4], additional_coinbase_value=1)
+ self.sync_blocks([b9], False, 16, b'bad-cb-amount', reconnect=True)
# Create a fork that ends in a block with too much fee (the one that causes the reorg)
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b10 (3) -> b11 (4)
# \-> b3 (1) -> b4 (2)
- tip(5)
- block(10, spend=out[3])
- yield rejected()
-
- block(11, spend=out[4], additional_coinbase_value=1)
- yield rejected(RejectResult(16, b'bad-cb-amount'))
+ self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer")
+ self.move_tip(5)
+ b10 = self.next_block(10, spend=out[3])
+ self.sync_blocks([b10], False)
+ b11 = self.next_block(11, spend=out[4], additional_coinbase_value=1)
+ self.sync_blocks([b11], False, 16, b'bad-cb-amount', reconnect=True)
# Try again, but with a valid fork first
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b14 (5)
- # (b12 added last)
# \-> b3 (1) -> b4 (2)
- tip(5)
- b12 = block(12, spend=out[3])
- save_spendable_output()
- b13 = block(13, spend=out[4])
- # Deliver the block header for b12, and the block b13.
- # b13 should be accepted but the tip won't advance until b12 is delivered.
- yield TestInstance([[CBlockHeader(b12), None], [b13, False]])
-
- save_spendable_output()
- # b14 is invalid, but the node won't know that until it tries to connect
- # Tip still can't advance because b12 is missing
- block(14, spend=out[5], additional_coinbase_value=1)
- yield rejected()
-
- yield TestInstance([[b12, True, b13.sha256]]) # New tip should be b13.
+ self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer (on a forked chain)")
+ self.move_tip(5)
+ b12 = self.next_block(12, spend=out[3])
+ self.save_spendable_output()
+ b13 = self.next_block(13, spend=out[4])
+ self.save_spendable_output()
+ b14 = self.next_block(14, spend=out[5], additional_coinbase_value=1)
+ self.sync_blocks([b12, b13, b14], False, 16, b'bad-cb-amount', reconnect=True)
+
+ # New tip should be b13.
+ assert_equal(node.getbestblockhash(), b13.hash)
# Add a block with MAX_BLOCK_SIGOPS and one with one more sigop
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5) -> b16 (6)
# \-> b3 (1) -> b4 (2)
-
- # Test that a block with a lot of checksigs is okay
+ self.log.info("Accept a block with lots of checksigs")
lots_of_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS - 1))
- tip(13)
- block(15, spend=out[5], script=lots_of_checksigs)
- yield accepted()
- save_spendable_output()
+ self.move_tip(13)
+ b15 = self.next_block(15, spend=out[5], script=lots_of_checksigs)
+ self.save_spendable_output()
+ self.sync_blocks([b15], True)
-
- # Test that a block with too many checksigs is rejected
+ self.log.info("Reject a block with too many checksigs")
too_many_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS))
- block(16, spend=out[6], script=too_many_checksigs)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
-
+ b16 = self.next_block(16, spend=out[6], script=too_many_checksigs)
+ self.sync_blocks([b16], False, 16, b'bad-blk-sigops', reconnect=True)
# Attempt to spend a transaction created on a different fork
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5) -> b17 (b3.vtx[1])
# \-> b3 (1) -> b4 (2)
- tip(15)
- block(17, spend=txout_b3)
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ self.log.info("Reject a block with a spend from a re-org'ed out tx")
+ self.move_tip(15)
+ b17 = self.next_block(17, spend=txout_b3)
+ self.sync_blocks([b17], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# Attempt to spend a transaction created on a different fork (on a fork this time)
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5)
# \-> b18 (b3.vtx[1]) -> b19 (6)
# \-> b3 (1) -> b4 (2)
- tip(13)
- block(18, spend=txout_b3)
- yield rejected()
+ self.log.info("Reject a block with a spend from a re-org'ed out tx (on a forked chain)")
+ self.move_tip(13)
+ b18 = self.next_block(18, spend=txout_b3)
+ self.sync_blocks([b18], False)
- block(19, spend=out[6])
- yield rejected()
+ b19 = self.next_block(19, spend=out[6])
+ self.sync_blocks([b19], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# Attempt to spend a coinbase at depth too low
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5) -> b20 (7)
# \-> b3 (1) -> b4 (2)
- tip(15)
- block(20, spend=out[7])
- yield rejected(RejectResult(16, b'bad-txns-premature-spend-of-coinbase'))
+ self.log.info("Reject a block spending an immature coinbase.")
+ self.move_tip(15)
+ b20 = self.next_block(20, spend=out[7])
+ self.sync_blocks([b20], False, 16, b'bad-txns-premature-spend-of-coinbase')
# Attempt to spend a coinbase at depth too low (on a fork this time)
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5)
# \-> b21 (6) -> b22 (5)
# \-> b3 (1) -> b4 (2)
- tip(13)
- block(21, spend=out[6])
- yield rejected()
+ self.log.info("Reject a block spending an immature coinbase (on a forked chain)")
+ self.move_tip(13)
+ b21 = self.next_block(21, spend=out[6])
+ self.sync_blocks([b21], False)
- block(22, spend=out[5])
- yield rejected()
+ b22 = self.next_block(22, spend=out[5])
+ self.sync_blocks([b22], False, 16, b'bad-txns-premature-spend-of-coinbase')
# Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6)
# \-> b24 (6) -> b25 (7)
# \-> b3 (1) -> b4 (2)
- tip(15)
- b23 = block(23, spend=out[6])
+ self.log.info("Accept a block of size MAX_BLOCK_BASE_SIZE")
+ self.move_tip(15)
+ b23 = self.next_block(23, spend=out[6])
tx = CTransaction()
script_length = MAX_BLOCK_BASE_SIZE - len(b23.serialize()) - 69
script_output = CScript([b'\x00' * script_length])
tx.vout.append(CTxOut(0, script_output))
tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 0)))
- b23 = update_block(23, [tx])
+ b23 = self.update_block(23, [tx])
# Make sure the math above worked out to produce a max-sized block
assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE)
- yield accepted()
- save_spendable_output()
+ self.sync_blocks([b23], True)
+ self.save_spendable_output()
- # Make the next block one byte bigger and check that it fails
- tip(15)
- b24 = block(24, spend=out[6])
+ self.log.info("Reject a block of size MAX_BLOCK_BASE_SIZE + 1")
+ self.move_tip(15)
+ b24 = self.next_block(24, spend=out[6])
script_length = MAX_BLOCK_BASE_SIZE - len(b24.serialize()) - 69
- script_output = CScript([b'\x00' * (script_length+1)])
+ script_output = CScript([b'\x00' * (script_length + 1)])
tx.vout = [CTxOut(0, script_output)]
- b24 = update_block(24, [tx])
- assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE+1)
- yield rejected(RejectResult(16, b'bad-blk-length'))
+ b24 = self.update_block(24, [tx])
+ assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE + 1)
+ self.sync_blocks([b24], False, 16, b'bad-blk-length', reconnect=True)
- block(25, spend=out[7])
- yield rejected()
+ b25 = self.next_block(25, spend=out[7])
+ self.sync_blocks([b25], False)
# Create blocks with a coinbase input script size out of range
# genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3)
# \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) -> b30 (7)
# \-> ... (6) -> ... (7)
# \-> b3 (1) -> b4 (2)
- tip(15)
- b26 = block(26, spend=out[6])
+ self.log.info("Reject a block with coinbase input script size out of range")
+ self.move_tip(15)
+ b26 = self.next_block(26, spend=out[6])
b26.vtx[0].vin[0].scriptSig = b'\x00'
b26.vtx[0].rehash()
# update_block causes the merkle root to get updated, even with no new
# transactions, and updates the required state.
- b26 = update_block(26, [])
- yield rejected(RejectResult(16, b'bad-cb-length'))
+ b26 = self.update_block(26, [])
+ self.sync_blocks([b26], False, 16, b'bad-cb-length', reconnect=True)
# Extend the b26 chain to make sure bitcoind isn't accepting b26
- block(27, spend=out[7])
- yield rejected(False)
+ b27 = self.next_block(27, spend=out[7])
+ self.sync_blocks([b27], False)
# Now try a too-large-coinbase script
- tip(15)
- b28 = block(28, spend=out[6])
+ self.move_tip(15)
+ b28 = self.next_block(28, spend=out[6])
b28.vtx[0].vin[0].scriptSig = b'\x00' * 101
b28.vtx[0].rehash()
- b28 = update_block(28, [])
- yield rejected(RejectResult(16, b'bad-cb-length'))
+ b28 = self.update_block(28, [])
+ self.sync_blocks([b28], False, 16, b'bad-cb-length', reconnect=True)
# Extend the b28 chain to make sure bitcoind isn't accepting b28
- block(29, spend=out[7])
- yield rejected(False)
+ b29 = self.next_block(29, spend=out[7])
+ self.sync_blocks([b29], False)
# b30 has a max-sized coinbase scriptSig.
- tip(23)
- b30 = block(30)
+ self.move_tip(23)
+ b30 = self.next_block(30)
b30.vtx[0].vin[0].scriptSig = b'\x00' * 100
b30.vtx[0].rehash()
- b30 = update_block(30, [])
- yield accepted()
- save_spendable_output()
+ b30 = self.update_block(30, [])
+ self.sync_blocks([b30], True)
+ self.save_spendable_output()
# b31 - b35 - check sigops of OP_CHECKMULTISIG / OP_CHECKMULTISIGVERIFY / OP_CHECKSIGVERIFY
#
@@ -433,42 +347,45 @@ class FullBlockTest(ComparisonTestFramework):
#
# MULTISIG: each op code counts as 20 sigops. To create the edge case, pack another 19 sigops at the end.
- lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19)
- b31 = block(31, spend=out[8], script=lots_of_multisigs)
+ self.log.info("Accept a block with the max number of OP_CHECKMULTISIG sigops")
+ lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19)
+ b31 = self.next_block(31, spend=out[8], script=lots_of_multisigs)
assert_equal(get_legacy_sigopcount_block(b31), MAX_BLOCK_SIGOPS)
- yield accepted()
- save_spendable_output()
+ self.sync_blocks([b31], True)
+ self.save_spendable_output()
# this goes over the limit because the coinbase has one sigop
+ self.log.info("Reject a block with too many OP_CHECKMULTISIG sigops")
too_many_multisigs = CScript([OP_CHECKMULTISIG] * (MAX_BLOCK_SIGOPS // 20))
- b32 = block(32, spend=out[9], script=too_many_multisigs)
+ b32 = self.next_block(32, spend=out[9], script=too_many_multisigs)
assert_equal(get_legacy_sigopcount_block(b32), MAX_BLOCK_SIGOPS + 1)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
-
+ self.sync_blocks([b32], False, 16, b'bad-blk-sigops', reconnect=True)
# CHECKMULTISIGVERIFY
- tip(31)
- lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19)
- block(33, spend=out[9], script=lots_of_multisigs)
- yield accepted()
- save_spendable_output()
-
+ self.log.info("Accept a block with the max number of OP_CHECKMULTISIGVERIFY sigops")
+ self.move_tip(31)
+ lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19)
+ b33 = self.next_block(33, spend=out[9], script=lots_of_multisigs)
+ self.sync_blocks([b33], True)
+ self.save_spendable_output()
+
+ self.log.info("Reject a block with too many OP_CHECKMULTISIGVERIFY sigops")
too_many_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * (MAX_BLOCK_SIGOPS // 20))
- block(34, spend=out[10], script=too_many_multisigs)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
-
+ b34 = self.next_block(34, spend=out[10], script=too_many_multisigs)
+ self.sync_blocks([b34], False, 16, b'bad-blk-sigops', reconnect=True)
# CHECKSIGVERIFY
- tip(33)
+ self.log.info("Accept a block with the max number of OP_CHECKSIGVERIFY sigops")
+ self.move_tip(33)
lots_of_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS - 1))
- b35 = block(35, spend=out[10], script=lots_of_checksigs)
- yield accepted()
- save_spendable_output()
+ b35 = self.next_block(35, spend=out[10], script=lots_of_checksigs)
+ self.sync_blocks([b35], True)
+ self.save_spendable_output()
+ self.log.info("Reject a block with too many OP_CHECKSIGVERIFY sigops")
too_many_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS))
- block(36, spend=out[11], script=too_many_checksigs)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
-
+ b36 = self.next_block(36, spend=out[11], script=too_many_checksigs)
+ self.sync_blocks([b36], False, 16, b'bad-blk-sigops', reconnect=True)
# Check spending of a transaction in a block which failed to connect
#
@@ -479,17 +396,18 @@ class FullBlockTest(ComparisonTestFramework):
#
# save 37's spendable output, but then double-spend out11 to invalidate the block
- tip(35)
- b37 = block(37, spend=out[11])
+ self.log.info("Reject a block spending transaction from a block which failed to connect")
+ self.move_tip(35)
+ b37 = self.next_block(37, spend=out[11])
txout_b37 = PreviousSpendableOutput(b37.vtx[1], 0)
- tx = create_and_sign_tx(out[11].tx, out[11].n, 0)
- b37 = update_block(37, [tx])
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ tx = self.create_and_sign_transaction(out[11].tx, out[11].n, 0)
+ b37 = self.update_block(37, [tx])
+ self.sync_blocks([b37], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# attempt to spend b37's first non-coinbase tx, at which point b37 was still considered valid
- tip(35)
- block(38, spend=txout_b37)
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ self.move_tip(35)
+ b38 = self.next_block(38, spend=txout_b37)
+ self.sync_blocks([b38], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# Check P2SH SigOp counting
#
@@ -502,45 +420,45 @@ class FullBlockTest(ComparisonTestFramework):
# redeem_script = COINBASE_PUBKEY, (OP_2DUP+OP_CHECKSIGVERIFY) * 5, OP_CHECKSIG
# p2sh_script = OP_HASH160, ripemd160(sha256(script)), OP_EQUAL
#
- tip(35)
- b39 = block(39)
+ self.log.info("Check P2SH SIGOPS are correctly counted")
+ self.move_tip(35)
+ b39 = self.next_block(39)
b39_outputs = 0
b39_sigops_per_output = 6
# Build the redeem script, hash it, use hash to create the p2sh script
- redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY]*5 + [OP_CHECKSIG])
+ redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY] * 5 + [OP_CHECKSIG])
redeem_script_hash = hash160(redeem_script)
p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL])
# Create a transaction that spends one satoshi to the p2sh_script, the rest to OP_TRUE
# This must be signed because it is spending a coinbase
spend = out[11]
- tx = create_tx(spend.tx, spend.n, 1, p2sh_script)
+ tx = self.create_tx(spend.tx, spend.n, 1, p2sh_script)
tx.vout.append(CTxOut(spend.tx.vout[spend.n].nValue - 1, CScript([OP_TRUE])))
self.sign_tx(tx, spend.tx, spend.n)
tx.rehash()
- b39 = update_block(39, [tx])
+ b39 = self.update_block(39, [tx])
b39_outputs += 1
# Until block is full, add tx's with 1 satoshi to p2sh_script, the rest to OP_TRUE
tx_new = None
tx_last = tx
- total_size=len(b39.serialize())
+ total_size = len(b39.serialize())
while(total_size < MAX_BLOCK_BASE_SIZE):
- tx_new = create_tx(tx_last, 1, 1, p2sh_script)
+ tx_new = self.create_tx(tx_last, 1, 1, p2sh_script)
tx_new.vout.append(CTxOut(tx_last.vout[1].nValue - 1, CScript([OP_TRUE])))
tx_new.rehash()
total_size += len(tx_new.serialize())
if total_size >= MAX_BLOCK_BASE_SIZE:
break
- b39.vtx.append(tx_new) # add tx to block
+ b39.vtx.append(tx_new) # add tx to block
tx_last = tx_new
b39_outputs += 1
- b39 = update_block(39, [])
- yield accepted()
- save_spendable_output()
-
+ b39 = self.update_block(39, [])
+ self.sync_blocks([b39], True)
+ self.save_spendable_output()
# Test sigops in P2SH redeem scripts
#
@@ -549,15 +467,16 @@ class FullBlockTest(ComparisonTestFramework):
#
# b41 does the same, less one, so it has the maximum sigops permitted.
#
- tip(39)
- b40 = block(40, spend=out[12])
+ self.log.info("Reject a block with too many P2SH sigops")
+ self.move_tip(39)
+ b40 = self.next_block(40, spend=out[12])
sigops = get_legacy_sigopcount_block(b40)
numTxes = (MAX_BLOCK_SIGOPS - sigops) // b39_sigops_per_output
assert_equal(numTxes <= b39_outputs, True)
lastOutpoint = COutPoint(b40.vtx[1].sha256, 0)
new_txs = []
- for i in range(1, numTxes+1):
+ for i in range(1, numTxes + 1):
tx = CTransaction()
tx.vout.append(CTxOut(1, CScript([OP_TRUE])))
tx.vin.append(CTxIn(lastOutpoint, b''))
@@ -579,35 +498,34 @@ class FullBlockTest(ComparisonTestFramework):
tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b40_sigops_to_fill)))
tx.rehash()
new_txs.append(tx)
- update_block(40, new_txs)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
+ self.update_block(40, new_txs)
+ self.sync_blocks([b40], False, 16, b'bad-blk-sigops', reconnect=True)
# same as b40, but one less sigop
- tip(39)
- block(41, spend=None)
- update_block(41, b40.vtx[1:-1])
+ self.log.info("Accept a block with the max number of P2SH sigops")
+ self.move_tip(39)
+ b41 = self.next_block(41, spend=None)
+ self.update_block(41, b40.vtx[1:-1])
b41_sigops_to_fill = b40_sigops_to_fill - 1
tx = CTransaction()
tx.vin.append(CTxIn(lastOutpoint, b''))
tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b41_sigops_to_fill)))
tx.rehash()
- update_block(41, [tx])
- yield accepted()
+ self.update_block(41, [tx])
+ self.sync_blocks([b41], True)
# Fork off of b39 to create a constant base again
#
# b23 (6) -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13)
# \-> b41 (12)
#
- tip(39)
- block(42, spend=out[12])
- yield rejected()
- save_spendable_output()
-
- block(43, spend=out[13])
- yield accepted()
- save_spendable_output()
+ self.move_tip(39)
+ b42 = self.next_block(42, spend=out[12])
+ self.save_spendable_output()
+ b43 = self.next_block(43, spend=out[13])
+ self.save_spendable_output()
+ self.sync_blocks([b42, b43], True)
# Test a number of really invalid scenarios
#
@@ -616,6 +534,7 @@ class FullBlockTest(ComparisonTestFramework):
# The next few blocks are going to be created "by hand" since they'll do funky things, such as having
# the first transaction be non-coinbase, etc. The purpose of b44 is to make sure this works.
+ self.log.info("Build block 44 manually")
height = self.block_heights[self.tip.sha256] + 1
coinbase = create_coinbase(height, self.coinbase_pubkey)
b44 = CBlock()
@@ -628,10 +547,10 @@ class FullBlockTest(ComparisonTestFramework):
self.tip = b44
self.block_heights[b44.sha256] = height
self.blocks[44] = b44
- yield accepted()
+ self.sync_blocks([b44], True)
- # A block with a non-coinbase as the first tx
- non_coinbase = create_tx(out[15].tx, out[15].n, 1)
+ self.log.info("Reject a block with a non-coinbase as the first tx")
+ non_coinbase = self.create_tx(out[15].tx, out[15].n, 1)
b45 = CBlock()
b45.nTime = self.tip.nTime + 1
b45.hashPrevBlock = self.tip.sha256
@@ -640,104 +559,102 @@ class FullBlockTest(ComparisonTestFramework):
b45.hashMerkleRoot = b45.calc_merkle_root()
b45.calc_sha256()
b45.solve()
- self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256]+1
+ self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256] + 1
self.tip = b45
self.blocks[45] = b45
- yield rejected(RejectResult(16, b'bad-cb-missing'))
+ self.sync_blocks([b45], False, 16, b'bad-cb-missing', reconnect=True)
- # A block with no txns
- tip(44)
+ self.log.info("Reject a block with no transactions")
+ self.move_tip(44)
b46 = CBlock()
- b46.nTime = b44.nTime+1
+ b46.nTime = b44.nTime + 1
b46.hashPrevBlock = b44.sha256
b46.nBits = 0x207fffff
b46.vtx = []
b46.hashMerkleRoot = 0
b46.solve()
- self.block_heights[b46.sha256] = self.block_heights[b44.sha256]+1
+ self.block_heights[b46.sha256] = self.block_heights[b44.sha256] + 1
self.tip = b46
assert 46 not in self.blocks
self.blocks[46] = b46
- s = ser_uint256(b46.hashMerkleRoot)
- yield rejected(RejectResult(16, b'bad-blk-length'))
+ self.sync_blocks([b46], False, 16, b'bad-blk-length', reconnect=True)
- # A block with invalid work
- tip(44)
- b47 = block(47, solve=False)
+ self.log.info("Reject a block with invalid work")
+ self.move_tip(44)
+ b47 = self.next_block(47, solve=False)
target = uint256_from_compact(b47.nBits)
- while b47.sha256 < target: #changed > to <
+ while b47.sha256 < target:
b47.nNonce += 1
b47.rehash()
- yield rejected(RejectResult(16, b'high-hash'))
+ self.sync_blocks([b47], False, request_block=False)
- # A block with timestamp > 2 hrs in the future
- tip(44)
- b48 = block(48, solve=False)
+ self.log.info("Reject a block with a timestamp >2 hours in the future")
+ self.move_tip(44)
+ b48 = self.next_block(48, solve=False)
b48.nTime = int(time.time()) + 60 * 60 * 3
b48.solve()
- yield rejected(RejectResult(16, b'time-too-new'))
+ self.sync_blocks([b48], False, request_block=False)
- # A block with an invalid merkle hash
- tip(44)
- b49 = block(49)
+ self.log.info("Reject a block with invalid merkle hash")
+ self.move_tip(44)
+ b49 = self.next_block(49)
b49.hashMerkleRoot += 1
b49.solve()
- yield rejected(RejectResult(16, b'bad-txnmrklroot'))
+ self.sync_blocks([b49], False, 16, b'bad-txnmrklroot', reconnect=True)
- # A block with an incorrect POW limit
- tip(44)
- b50 = block(50)
+ self.log.info("Reject a block with incorrect POW limit")
+ self.move_tip(44)
+ b50 = self.next_block(50)
b50.nBits = b50.nBits - 1
b50.solve()
- yield rejected(RejectResult(16, b'bad-diffbits'))
+ self.sync_blocks([b50], False, request_block=False, reconnect=True)
- # A block with two coinbase txns
- tip(44)
- b51 = block(51)
+ self.log.info("Reject a block with two coinbase transactions")
+ self.move_tip(44)
+ b51 = self.next_block(51)
cb2 = create_coinbase(51, self.coinbase_pubkey)
- b51 = update_block(51, [cb2])
- yield rejected(RejectResult(16, b'bad-cb-multiple'))
+ b51 = self.update_block(51, [cb2])
+ self.sync_blocks([b51], False, 16, b'bad-cb-multiple', reconnect=True)
- # A block w/ duplicate txns
+ self.log.info("Reject a block with duplicate transactions")
# Note: txns have to be in the right position in the merkle tree to trigger this error
- tip(44)
- b52 = block(52, spend=out[15])
- tx = create_tx(b52.vtx[1], 0, 1)
- b52 = update_block(52, [tx, tx])
- yield rejected(RejectResult(16, b'bad-txns-duplicate'))
+ self.move_tip(44)
+ b52 = self.next_block(52, spend=out[15])
+ tx = self.create_tx(b52.vtx[1], 0, 1)
+ b52 = self.update_block(52, [tx, tx])
+ self.sync_blocks([b52], False, 16, b'bad-txns-duplicate', reconnect=True)
# Test block timestamps
# -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15)
# \-> b54 (15)
#
- tip(43)
- block(53, spend=out[14])
- yield rejected() # rejected since b44 is at same height
- save_spendable_output()
+ self.move_tip(43)
+ b53 = self.next_block(53, spend=out[14])
+ self.sync_blocks([b53], False)
+ self.save_spendable_output()
- # invalid timestamp (b35 is 5 blocks back, so its time is MedianTimePast)
- b54 = block(54, spend=out[15])
+ self.log.info("Reject a block with timestamp before MedianTimePast")
+ b54 = self.next_block(54, spend=out[15])
b54.nTime = b35.nTime - 1
b54.solve()
- yield rejected(RejectResult(16, b'time-too-old'))
+ self.sync_blocks([b54], False, request_block=False)
# valid timestamp
- tip(53)
- b55 = block(55, spend=out[15])
+ self.move_tip(53)
+ b55 = self.next_block(55, spend=out[15])
b55.nTime = b35.nTime
- update_block(55, [])
- yield accepted()
- save_spendable_output()
+ self.update_block(55, [])
+ self.sync_blocks([b55], True)
+ self.save_spendable_output()
-
- # Test CVE-2012-2459
+ # Test Merkle tree malleability
#
# -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57p2 (16)
# \-> b57 (16)
# \-> b56p2 (16)
# \-> b56 (16)
#
- # Merkle tree malleability (CVE-2012-2459): repeating sequences of transactions in a block without
+ # Merkle tree malleability (CVE-2012-2459): repeating sequences of transactions in a block without
# affecting the merkle root of a block, while still invalidating it.
# See: src/consensus/merkle.h
#
@@ -758,46 +675,48 @@ class FullBlockTest(ComparisonTestFramework):
# that the error was caught early, avoiding a DOS vulnerability.)
# b57 - a good block with 2 txs, don't submit until end
- tip(55)
- b57 = block(57)
- tx = create_and_sign_tx(out[16].tx, out[16].n, 1)
- tx1 = create_tx(tx, 0, 1)
- b57 = update_block(57, [tx, tx1])
+ self.move_tip(55)
+ b57 = self.next_block(57)
+ tx = self.create_and_sign_transaction(out[16].tx, out[16].n, 1)
+ tx1 = self.create_tx(tx, 0, 1)
+ b57 = self.update_block(57, [tx, tx1])
# b56 - copy b57, add a duplicate tx
- tip(55)
+ self.log.info("Reject a block with a duplicate transaction in the Merkle Tree (but with a valid Merkle Root)")
+ self.move_tip(55)
b56 = copy.deepcopy(b57)
self.blocks[56] = b56
- assert_equal(len(b56.vtx),3)
- b56 = update_block(56, [tx1])
+ assert_equal(len(b56.vtx), 3)
+ b56 = self.update_block(56, [tx1])
assert_equal(b56.hash, b57.hash)
- yield rejected(RejectResult(16, b'bad-txns-duplicate'))
+ self.sync_blocks([b56], False, 16, b'bad-txns-duplicate', reconnect=True)
# b57p2 - a good block with 6 tx'es, don't submit until end
- tip(55)
- b57p2 = block("57p2")
- tx = create_and_sign_tx(out[16].tx, out[16].n, 1)
- tx1 = create_tx(tx, 0, 1)
- tx2 = create_tx(tx1, 0, 1)
- tx3 = create_tx(tx2, 0, 1)
- tx4 = create_tx(tx3, 0, 1)
- b57p2 = update_block("57p2", [tx, tx1, tx2, tx3, tx4])
+ self.move_tip(55)
+ b57p2 = self.next_block("57p2")
+ tx = self.create_and_sign_transaction(out[16].tx, out[16].n, 1)
+ tx1 = self.create_tx(tx, 0, 1)
+ tx2 = self.create_tx(tx1, 0, 1)
+ tx3 = self.create_tx(tx2, 0, 1)
+ tx4 = self.create_tx(tx3, 0, 1)
+ b57p2 = self.update_block("57p2", [tx, tx1, tx2, tx3, tx4])
# b56p2 - copy b57p2, duplicate two non-consecutive tx's
- tip(55)
+ self.log.info("Reject a block with two duplicate transactions in the Merkle Tree (but with a valid Merkle Root)")
+ self.move_tip(55)
b56p2 = copy.deepcopy(b57p2)
self.blocks["b56p2"] = b56p2
assert_equal(b56p2.hash, b57p2.hash)
- assert_equal(len(b56p2.vtx),6)
- b56p2 = update_block("b56p2", [tx3, tx4])
- yield rejected(RejectResult(16, b'bad-txns-duplicate'))
+ assert_equal(len(b56p2.vtx), 6)
+ b56p2 = self.update_block("b56p2", [tx3, tx4])
+ self.sync_blocks([b56p2], False, 16, b'bad-txns-duplicate', reconnect=True)
- tip("57p2")
- yield accepted()
+ self.move_tip("57p2")
+ self.sync_blocks([b57p2], True)
- tip(57)
- yield rejected() #rejected because 57p2 seen first
- save_spendable_output()
+ self.move_tip(57)
+ self.sync_blocks([b57], False) # The tip is not updated because 57p2 seen first
+ self.save_spendable_output()
# Test a few invalid tx types
#
@@ -806,28 +725,30 @@ class FullBlockTest(ComparisonTestFramework):
#
# tx with prevout.n out of range
- tip(57)
- b58 = block(58, spend=out[17])
+ self.log.info("Reject a block with a transaction with prevout.n out of range")
+ self.move_tip(57)
+ b58 = self.next_block(58, spend=out[17])
tx = CTransaction()
assert(len(out[17].tx.vout) < 42)
tx.vin.append(CTxIn(COutPoint(out[17].tx.sha256, 42), CScript([OP_TRUE]), 0xffffffff))
tx.vout.append(CTxOut(0, b""))
tx.calc_sha256()
- b58 = update_block(58, [tx])
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ b58 = self.update_block(58, [tx])
+ self.sync_blocks([b58], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
- # tx with output value > input value out of range
- tip(57)
- b59 = block(59)
- tx = create_and_sign_tx(out[17].tx, out[17].n, 51*COIN)
- b59 = update_block(59, [tx])
- yield rejected(RejectResult(16, b'bad-txns-in-belowout'))
+ # tx with output value > input value
+ self.log.info("Reject a block with a transaction with outputs > inputs")
+ self.move_tip(57)
+ b59 = self.next_block(59)
+ tx = self.create_and_sign_transaction(out[17].tx, out[17].n, 51 * COIN)
+ b59 = self.update_block(59, [tx])
+ self.sync_blocks([b59], False, 16, b'bad-txns-in-belowout', reconnect=True)
# reset to good chain
- tip(57)
- b60 = block(60, spend=out[17])
- yield accepted()
- save_spendable_output()
+ self.move_tip(57)
+ b60 = self.next_block(60, spend=out[17])
+ self.sync_blocks([b60], True)
+ self.save_spendable_output()
# Test BIP30
#
@@ -838,46 +759,46 @@ class FullBlockTest(ComparisonTestFramework):
# not-fully-spent transaction in the same chain. To test, make identical coinbases;
# the second one should be rejected.
#
- tip(60)
- b61 = block(61, spend=out[18])
- b61.vtx[0].vin[0].scriptSig = b60.vtx[0].vin[0].scriptSig #equalize the coinbases
+ self.log.info("Reject a block with a transaction with a duplicate hash of a previous transaction (BIP30)")
+ self.move_tip(60)
+ b61 = self.next_block(61, spend=out[18])
+ b61.vtx[0].vin[0].scriptSig = b60.vtx[0].vin[0].scriptSig # Equalize the coinbases
b61.vtx[0].rehash()
- b61 = update_block(61, [])
+ b61 = self.update_block(61, [])
assert_equal(b60.vtx[0].serialize(), b61.vtx[0].serialize())
- yield rejected(RejectResult(16, b'bad-txns-BIP30'))
-
+ self.sync_blocks([b61], False, 16, b'bad-txns-BIP30', reconnect=True)
# Test tx.isFinal is properly rejected (not an exhaustive tx.isFinal test, that should be in data-driven transaction tests)
#
# -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17)
# \-> b62 (18)
#
- tip(60)
- b62 = block(62)
+ self.log.info("Reject a block with a transaction with a nonfinal locktime")
+ self.move_tip(60)
+ b62 = self.next_block(62)
tx = CTransaction()
- tx.nLockTime = 0xffffffff #this locktime is non-final
+ tx.nLockTime = 0xffffffff # this locktime is non-final
assert(out[18].n < len(out[18].tx.vout))
- tx.vin.append(CTxIn(COutPoint(out[18].tx.sha256, out[18].n))) # don't set nSequence
+ tx.vin.append(CTxIn(COutPoint(out[18].tx.sha256, out[18].n))) # don't set nSequence
tx.vout.append(CTxOut(0, CScript([OP_TRUE])))
assert(tx.vin[0].nSequence < 0xffffffff)
tx.calc_sha256()
- b62 = update_block(62, [tx])
- yield rejected(RejectResult(16, b'bad-txns-nonfinal'))
-
+ b62 = self.update_block(62, [tx])
+ self.sync_blocks([b62], False, 16, b'bad-txns-nonfinal')
# Test a non-final coinbase is also rejected
#
# -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17)
# \-> b63 (-)
#
- tip(60)
- b63 = block(63)
+ self.log.info("Reject a block with a coinbase transaction with a nonfinal locktime")
+ self.move_tip(60)
+ b63 = self.next_block(63)
b63.vtx[0].nLockTime = 0xffffffff
b63.vtx[0].vin[0].nSequence = 0xDEADBEEF
b63.vtx[0].rehash()
- b63 = update_block(63, [])
- yield rejected(RejectResult(16, b'bad-txns-nonfinal'))
-
+ b63 = self.update_block(63, [])
+ self.sync_blocks([b63], False, 16, b'bad-txns-nonfinal')
# This checks that a block with a bloated VARINT between the block_header and the array of tx such that
# the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint,
@@ -893,8 +814,9 @@ class FullBlockTest(ComparisonTestFramework):
# b64a is a bloated block (non-canonical varint)
# b64 is a good block (same as b64 but w/ canonical varint)
#
- tip(60)
- regular_block = block("64a", spend=out[18])
+ self.log.info("Accept a valid block even if a bloated version of the block has previously been sent")
+ self.move_tip(60)
+ regular_block = self.next_block("64a", spend=out[18])
# make it a "broken_block," with non-canonical serialization
b64a = CBrokenBlock(regular_block)
@@ -908,45 +830,51 @@ class FullBlockTest(ComparisonTestFramework):
script_output = CScript([b'\x00' * script_length])
tx.vout.append(CTxOut(0, script_output))
tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0)))
- b64a = update_block("64a", [tx])
+ b64a = self.update_block("64a", [tx])
assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8)
- yield TestInstance([[self.tip, None]])
+ self.sync_blocks([b64a], False, 1, b'error parsing message')
- # comptool workaround: to make sure b64 is delivered, manually erase b64a from blockstore
- self.test.block_store.erase(b64a.sha256)
+ # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently
+ # resend the header message, it won't send us the getdata message again. Just
+ # disconnect and reconnect and then call sync_blocks.
+ # TODO: improve this test to be less dependent on P2P DOS behaviour.
+ node.disconnect_p2ps()
+ self.reconnect_p2p()
- tip(60)
+ self.move_tip(60)
b64 = CBlock(b64a)
b64.vtx = copy.deepcopy(b64a.vtx)
assert_equal(b64.hash, b64a.hash)
assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE)
self.blocks[64] = b64
- update_block(64, [])
- yield accepted()
- save_spendable_output()
+ b64 = self.update_block(64, [])
+ self.sync_blocks([b64], True)
+ self.save_spendable_output()
# Spend an output created in the block itself
#
# -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19)
#
- tip(64)
- block(65)
- tx1 = create_and_sign_tx(out[19].tx, out[19].n, out[19].tx.vout[0].nValue)
- tx2 = create_and_sign_tx(tx1, 0, 0)
- update_block(65, [tx1, tx2])
- yield accepted()
- save_spendable_output()
+ self.log.info("Accept a block with a transaction spending an output created in the same block")
+ self.move_tip(64)
+ b65 = self.next_block(65)
+ tx1 = self.create_and_sign_transaction(out[19].tx, out[19].n, out[19].tx.vout[0].nValue)
+ tx2 = self.create_and_sign_transaction(tx1, 0, 0)
+ b65 = self.update_block(65, [tx1, tx2])
+ self.sync_blocks([b65], True)
+ self.save_spendable_output()
# Attempt to spend an output created later in the same block
#
# -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19)
# \-> b66 (20)
- tip(65)
- block(66)
- tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue)
- tx2 = create_and_sign_tx(tx1, 0, 1)
- update_block(66, [tx2, tx1])
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ self.log.info("Reject a block with a transaction spending an output created later in the same block")
+ self.move_tip(65)
+ b66 = self.next_block(66)
+ tx1 = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue)
+ tx2 = self.create_and_sign_transaction(tx1, 0, 1)
+ b66 = self.update_block(66, [tx2, tx1])
+ self.sync_blocks([b66], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# Attempt to double-spend a transaction created in a block
#
@@ -954,13 +882,14 @@ class FullBlockTest(ComparisonTestFramework):
# \-> b67 (20)
#
#
- tip(65)
- block(67)
- tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue)
- tx2 = create_and_sign_tx(tx1, 0, 1)
- tx3 = create_and_sign_tx(tx1, 0, 2)
- update_block(67, [tx1, tx2, tx3])
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
+ self.log.info("Reject a block with a transaction double spending a transaction creted in the same block")
+ self.move_tip(65)
+ b67 = self.next_block(67)
+ tx1 = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue)
+ tx2 = self.create_and_sign_transaction(tx1, 0, 1)
+ tx3 = self.create_and_sign_transaction(tx1, 0, 2)
+ b67 = self.update_block(67, [tx1, tx2, tx3])
+ self.sync_blocks([b67], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# More tests of block subsidy
#
@@ -974,34 +903,36 @@ class FullBlockTest(ComparisonTestFramework):
# b69 - coinbase with extra 10 satoshis, and a tx that gives a 10 satoshi fee
# this succeeds
#
- tip(65)
- block(68, additional_coinbase_value=10)
- tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-9)
- update_block(68, [tx])
- yield rejected(RejectResult(16, b'bad-cb-amount'))
-
- tip(65)
- b69 = block(69, additional_coinbase_value=10)
- tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-10)
- update_block(69, [tx])
- yield accepted()
- save_spendable_output()
+ self.log.info("Reject a block trying to claim too much subsidy in the coinbase transaction")
+ self.move_tip(65)
+ b68 = self.next_block(68, additional_coinbase_value=10)
+ tx = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue - 9)
+ b68 = self.update_block(68, [tx])
+ self.sync_blocks([b68], False, 16, b'bad-cb-amount', reconnect=True)
+
+ self.log.info("Accept a block claiming the correct subsidy in the coinbase transaction")
+ self.move_tip(65)
+ b69 = self.next_block(69, additional_coinbase_value=10)
+ tx = self.create_and_sign_transaction(out[20].tx, out[20].n, out[20].tx.vout[0].nValue - 10)
+ self.update_block(69, [tx])
+ self.sync_blocks([b69], True)
+ self.save_spendable_output()
# Test spending the outpoint of a non-existent transaction
#
# -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20)
# \-> b70 (21)
#
- tip(69)
- block(70, spend=out[21])
+ self.log.info("Reject a block containing a transaction spending from a non-existent input")
+ self.move_tip(69)
+ b70 = self.next_block(70, spend=out[21])
bogus_tx = CTransaction()
bogus_tx.sha256 = uint256_from_str(b"23c70ed7c0506e9178fc1a987f40a33946d4ad4c962b5ae3a52546da53af0c5c")
tx = CTransaction()
tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff))
tx.vout.append(CTxOut(1, b""))
- update_block(70, [tx])
- yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent'))
-
+ b70 = self.update_block(70, [tx])
+ self.sync_blocks([b70], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
# Test accepting an invalid block which has the same hash as a valid one (via merkle tree tricks)
#
@@ -1009,13 +940,13 @@ class FullBlockTest(ComparisonTestFramework):
# \-> b71 (21)
#
# b72 is a good block.
- # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b71.
- #
- tip(69)
- b72 = block(72)
- tx1 = create_and_sign_tx(out[21].tx, out[21].n, 2)
- tx2 = create_and_sign_tx(tx1, 0, 1)
- b72 = update_block(72, [tx1, tx2]) # now tip is 72
+ # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b72.
+ self.log.info("Reject a block containing a duplicate transaction but with the same Merkle root (Merkle tree malleability")
+ self.move_tip(69)
+ b72 = self.next_block(72)
+ tx1 = self.create_and_sign_transaction(out[21].tx, out[21].n, 2)
+ tx2 = self.create_and_sign_transaction(tx1, 0, 1)
+ b72 = self.update_block(72, [tx1, tx2]) # now tip is 72
b71 = copy.deepcopy(b72)
b71.vtx.append(tx2) # add duplicate tx2
self.block_heights[b71.sha256] = self.block_heights[b69.sha256] + 1 # b71 builds off b69
@@ -1025,12 +956,12 @@ class FullBlockTest(ComparisonTestFramework):
assert_equal(len(b72.vtx), 3)
assert_equal(b72.sha256, b71.sha256)
- tip(71)
- yield rejected(RejectResult(16, b'bad-txns-duplicate'))
- tip(72)
- yield accepted()
- save_spendable_output()
+ self.move_tip(71)
+ self.sync_blocks([b71], False, 16, b'bad-txns-duplicate', reconnect=True)
+ self.move_tip(72)
+ self.sync_blocks([b72], True)
+ self.save_spendable_output()
# Test some invalid scripts and MAX_BLOCK_SIGOPS
#
@@ -1048,23 +979,23 @@ class FullBlockTest(ComparisonTestFramework):
# bytearray[20,000-20,003]: 521 (max_script_element_size+1, in little-endian format)
# bytearray[20,004-20,525]: unread data (script_element)
# bytearray[20,526] : OP_CHECKSIG (this puts us over the limit)
- #
- tip(72)
- b73 = block(73)
+ self.log.info("Reject a block containing too many sigops after a large script element")
+ self.move_tip(72)
+ b73 = self.next_block(73)
size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 + 1
a = bytearray([OP_CHECKSIG] * size)
- a[MAX_BLOCK_SIGOPS - 1] = int("4e",16) # OP_PUSHDATA4
+ a[MAX_BLOCK_SIGOPS - 1] = int("4e", 16) # OP_PUSHDATA4
element_size = MAX_SCRIPT_ELEMENT_SIZE + 1
a[MAX_BLOCK_SIGOPS] = element_size % 256
- a[MAX_BLOCK_SIGOPS+1] = element_size // 256
- a[MAX_BLOCK_SIGOPS+2] = 0
- a[MAX_BLOCK_SIGOPS+3] = 0
+ a[MAX_BLOCK_SIGOPS + 1] = element_size // 256
+ a[MAX_BLOCK_SIGOPS + 2] = 0
+ a[MAX_BLOCK_SIGOPS + 3] = 0
- tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a))
- b73 = update_block(73, [tx])
- assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS+1)
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
+ tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a))
+ b73 = self.update_block(73, [tx])
+ assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS + 1)
+ self.sync_blocks([b73], False, 16, b'bad-blk-sigops', reconnect=True)
# b74/75 - if we push an invalid script element, all prevous sigops are counted,
# but sigops after the element are not counted.
@@ -1076,45 +1007,44 @@ class FullBlockTest(ComparisonTestFramework):
#
# b74 fails because we put MAX_BLOCK_SIGOPS+1 before the element
# b75 succeeds because we put MAX_BLOCK_SIGOPS before the element
- #
- #
- tip(72)
- b74 = block(74)
- size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561
+ self.log.info("Check sigops are counted correctly after an invalid script element")
+ self.move_tip(72)
+ b74 = self.next_block(74)
+ size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561
a = bytearray([OP_CHECKSIG] * size)
a[MAX_BLOCK_SIGOPS] = 0x4e
- a[MAX_BLOCK_SIGOPS+1] = 0xfe
- a[MAX_BLOCK_SIGOPS+2] = 0xff
- a[MAX_BLOCK_SIGOPS+3] = 0xff
- a[MAX_BLOCK_SIGOPS+4] = 0xff
- tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a))
- b74 = update_block(74, [tx])
- yield rejected(RejectResult(16, b'bad-blk-sigops'))
-
- tip(72)
- b75 = block(75)
+ a[MAX_BLOCK_SIGOPS + 1] = 0xfe
+ a[MAX_BLOCK_SIGOPS + 2] = 0xff
+ a[MAX_BLOCK_SIGOPS + 3] = 0xff
+ a[MAX_BLOCK_SIGOPS + 4] = 0xff
+ tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a))
+ b74 = self.update_block(74, [tx])
+ self.sync_blocks([b74], False, 16, b'bad-blk-sigops', reconnect=True)
+
+ self.move_tip(72)
+ b75 = self.next_block(75)
size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42
a = bytearray([OP_CHECKSIG] * size)
- a[MAX_BLOCK_SIGOPS-1] = 0x4e
+ a[MAX_BLOCK_SIGOPS - 1] = 0x4e
a[MAX_BLOCK_SIGOPS] = 0xff
- a[MAX_BLOCK_SIGOPS+1] = 0xff
- a[MAX_BLOCK_SIGOPS+2] = 0xff
- a[MAX_BLOCK_SIGOPS+3] = 0xff
- tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a))
- b75 = update_block(75, [tx])
- yield accepted()
- save_spendable_output()
+ a[MAX_BLOCK_SIGOPS + 1] = 0xff
+ a[MAX_BLOCK_SIGOPS + 2] = 0xff
+ a[MAX_BLOCK_SIGOPS + 3] = 0xff
+ tx = self.create_and_sign_transaction(out[22].tx, 0, 1, CScript(a))
+ b75 = self.update_block(75, [tx])
+ self.sync_blocks([b75], True)
+ self.save_spendable_output()
# Check that if we push an element filled with CHECKSIGs, they are not counted
- tip(75)
- b76 = block(76)
+ self.move_tip(75)
+ b76 = self.next_block(76)
size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5
a = bytearray([OP_CHECKSIG] * size)
- a[MAX_BLOCK_SIGOPS-1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs
- tx = create_and_sign_tx(out[23].tx, 0, 1, CScript(a))
- b76 = update_block(76, [tx])
- yield accepted()
- save_spendable_output()
+ a[MAX_BLOCK_SIGOPS - 1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs
+ tx = self.create_and_sign_transaction(out[23].tx, 0, 1, CScript(a))
+ b76 = self.update_block(76, [tx])
+ self.sync_blocks([b76], True)
+ self.save_spendable_output()
# Test transaction resurrection
#
@@ -1133,39 +1063,39 @@ class FullBlockTest(ComparisonTestFramework):
# To get around this issue, we construct transactions which are not signed and which
# spend to OP_TRUE. If the standard-ness rules change, this test would need to be
# updated. (Perhaps to spend to a P2SH OP_TRUE script)
- #
- tip(76)
- block(77)
- tx77 = create_and_sign_tx(out[24].tx, out[24].n, 10*COIN)
- update_block(77, [tx77])
- yield accepted()
- save_spendable_output()
-
- block(78)
- tx78 = create_tx(tx77, 0, 9*COIN)
- update_block(78, [tx78])
- yield accepted()
-
- block(79)
- tx79 = create_tx(tx78, 0, 8*COIN)
- update_block(79, [tx79])
- yield accepted()
+ self.log.info("Test transaction resurrection during a re-org")
+ self.move_tip(76)
+ b77 = self.next_block(77)
+ tx77 = self.create_and_sign_transaction(out[24].tx, out[24].n, 10 * COIN)
+ b77 = self.update_block(77, [tx77])
+ self.sync_blocks([b77], True)
+ self.save_spendable_output()
+
+ b78 = self.next_block(78)
+ tx78 = self.create_tx(tx77, 0, 9 * COIN)
+ b78 = self.update_block(78, [tx78])
+ self.sync_blocks([b78], True)
+
+ b79 = self.next_block(79)
+ tx79 = self.create_tx(tx78, 0, 8 * COIN)
+ b79 = self.update_block(79, [tx79])
+ self.sync_blocks([b79], True)
# mempool should be empty
assert_equal(len(self.nodes[0].getrawmempool()), 0)
- tip(77)
- block(80, spend=out[25])
- yield rejected()
- save_spendable_output()
+ self.move_tip(77)
+ b80 = self.next_block(80, spend=out[25])
+ self.sync_blocks([b80], False, request_block=False)
+ self.save_spendable_output()
- block(81, spend=out[26])
- yield rejected() # other chain is same length
- save_spendable_output()
+ b81 = self.next_block(81, spend=out[26])
+ self.sync_blocks([b81], False, request_block=False) # other chain is same length
+ self.save_spendable_output()
- block(82, spend=out[27])
- yield accepted() # now this chain is longer, triggers re-org
- save_spendable_output()
+ b82 = self.next_block(82, spend=out[27])
+ self.sync_blocks([b82], True) # now this chain is longer, triggers re-org
+ self.save_spendable_output()
# now check that tx78 and tx79 have been put back into the peer's mempool
mempool = self.nodes[0].getrawmempool()
@@ -1173,33 +1103,32 @@ class FullBlockTest(ComparisonTestFramework):
assert(tx78.hash in mempool)
assert(tx79.hash in mempool)
-
# Test invalid opcodes in dead execution paths.
#
# -> b81 (26) -> b82 (27) -> b83 (28)
#
- block(83)
+ self.log.info("Accept a block with invalid opcodes in dead execution paths")
+ b83 = self.next_block(83)
op_codes = [OP_IF, OP_INVALIDOPCODE, OP_ELSE, OP_TRUE, OP_ENDIF]
script = CScript(op_codes)
- tx1 = create_and_sign_tx(out[28].tx, out[28].n, out[28].tx.vout[0].nValue, script)
+ tx1 = self.create_and_sign_transaction(out[28].tx, out[28].n, out[28].tx.vout[0].nValue, script)
- tx2 = create_and_sign_tx(tx1, 0, 0, CScript([OP_TRUE]))
+ tx2 = self.create_and_sign_transaction(tx1, 0, 0, CScript([OP_TRUE]))
tx2.vin[0].scriptSig = CScript([OP_FALSE])
tx2.rehash()
- update_block(83, [tx1, tx2])
- yield accepted()
- save_spendable_output()
-
+ b83 = self.update_block(83, [tx1, tx2])
+ self.sync_blocks([b83], True)
+ self.save_spendable_output()
# Reorg on/off blocks that have OP_RETURN in them (and try to spend them)
#
# -> b81 (26) -> b82 (27) -> b83 (28) -> b84 (29) -> b87 (30) -> b88 (31)
# \-> b85 (29) -> b86 (30) \-> b89a (32)
#
- #
- block(84)
- tx1 = create_tx(out[29].tx, out[29].n, 0, CScript([OP_RETURN]))
+ self.log.info("Test re-orging blocks with OP_RETURN in them")
+ b84 = self.next_block(84)
+ tx1 = self.create_tx(out[29].tx, out[29].n, 0, CScript([OP_RETURN]))
tx1.vout.append(CTxOut(0, CScript([OP_TRUE])))
tx1.vout.append(CTxOut(0, CScript([OP_TRUE])))
tx1.vout.append(CTxOut(0, CScript([OP_TRUE])))
@@ -1207,87 +1136,186 @@ class FullBlockTest(ComparisonTestFramework):
tx1.calc_sha256()
self.sign_tx(tx1, out[29].tx, out[29].n)
tx1.rehash()
- tx2 = create_tx(tx1, 1, 0, CScript([OP_RETURN]))
+ tx2 = self.create_tx(tx1, 1, 0, CScript([OP_RETURN]))
tx2.vout.append(CTxOut(0, CScript([OP_RETURN])))
- tx3 = create_tx(tx1, 2, 0, CScript([OP_RETURN]))
+ tx3 = self.create_tx(tx1, 2, 0, CScript([OP_RETURN]))
tx3.vout.append(CTxOut(0, CScript([OP_TRUE])))
- tx4 = create_tx(tx1, 3, 0, CScript([OP_TRUE]))
+ tx4 = self.create_tx(tx1, 3, 0, CScript([OP_TRUE]))
tx4.vout.append(CTxOut(0, CScript([OP_RETURN])))
- tx5 = create_tx(tx1, 4, 0, CScript([OP_RETURN]))
+ tx5 = self.create_tx(tx1, 4, 0, CScript([OP_RETURN]))
- update_block(84, [tx1,tx2,tx3,tx4,tx5])
- yield accepted()
- save_spendable_output()
+ b84 = self.update_block(84, [tx1, tx2, tx3, tx4, tx5])
+ self.sync_blocks([b84], True)
+ self.save_spendable_output()
- tip(83)
- block(85, spend=out[29])
- yield rejected()
+ self.move_tip(83)
+ b85 = self.next_block(85, spend=out[29])
+ self.sync_blocks([b85], False) # other chain is same length
- block(86, spend=out[30])
- yield accepted()
+ b86 = self.next_block(86, spend=out[30])
+ self.sync_blocks([b86], True)
- tip(84)
- block(87, spend=out[30])
- yield rejected()
- save_spendable_output()
+ self.move_tip(84)
+ b87 = self.next_block(87, spend=out[30])
+ self.sync_blocks([b87], False) # other chain is same length
+ self.save_spendable_output()
- block(88, spend=out[31])
- yield accepted()
- save_spendable_output()
+ b88 = self.next_block(88, spend=out[31])
+ self.sync_blocks([b88], True)
+ self.save_spendable_output()
# trying to spend the OP_RETURN output is rejected
- block("89a", spend=out[32])
- tx = create_tx(tx1, 0, 0, CScript([OP_TRUE]))
- update_block("89a", [tx])
- yield rejected()
-
-
- # Test re-org of a week's worth of blocks (1088 blocks)
- # This test takes a minute or two and can be accomplished in memory
- #
- if self.options.runbarelyexpensive:
- tip(88)
- LARGE_REORG_SIZE = 1088
- test1 = TestInstance(sync_every_block=False)
- spend=out[32]
- for i in range(89, LARGE_REORG_SIZE + 89):
- b = block(i, spend)
- tx = CTransaction()
- script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69
- script_output = CScript([b'\x00' * script_length])
- tx.vout.append(CTxOut(0, script_output))
- tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0)))
- b = update_block(i, [tx])
- assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE)
- test1.blocks_and_transactions.append([self.tip, True])
- save_spendable_output()
- spend = get_spendable_output()
-
- yield test1
- chain1_tip = i
-
- # now create alt chain of same length
- tip(88)
- test2 = TestInstance(sync_every_block=False)
- for i in range(89, LARGE_REORG_SIZE + 89):
- block("alt"+str(i))
- test2.blocks_and_transactions.append([self.tip, False])
- yield test2
-
- # extend alt chain to trigger re-org
- block("alt" + str(chain1_tip + 1))
- yield accepted()
-
- # ... and re-org back to the first chain
- tip(chain1_tip)
- block(chain1_tip + 1)
- yield rejected()
- block(chain1_tip + 2)
- yield accepted()
-
- chain1_tip += 2
+ b89a = self.next_block("89a", spend=out[32])
+ tx = self.create_tx(tx1, 0, 0, CScript([OP_TRUE]))
+ b89a = self.update_block("89a", [tx])
+ self.sync_blocks([b89a], False, 16, b'bad-txns-inputs-missingorspent', reconnect=True)
+
+ self.log.info("Test a re-org of one week's worth of blocks (1088 blocks)")
+
+ self.move_tip(88)
+ LARGE_REORG_SIZE = 1088
+ blocks = []
+ spend = out[32]
+ for i in range(89, LARGE_REORG_SIZE + 89):
+ b = self.next_block(i, spend)
+ tx = CTransaction()
+ script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69
+ script_output = CScript([b'\x00' * script_length])
+ tx.vout.append(CTxOut(0, script_output))
+ tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0)))
+ b = self.update_block(i, [tx])
+ assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE)
+ blocks.append(b)
+ self.save_spendable_output()
+ spend = self.get_spendable_output()
+
+ self.sync_blocks(blocks, True, timeout=180)
+ chain1_tip = i
+
+ # now create alt chain of same length
+ self.move_tip(88)
+ blocks2 = []
+ for i in range(89, LARGE_REORG_SIZE + 89):
+ blocks2.append(self.next_block("alt" + str(i)))
+ self.sync_blocks(blocks2, False, request_block=False)
+
+ # extend alt chain to trigger re-org
+ block = self.next_block("alt" + str(chain1_tip + 1))
+ self.sync_blocks([block], True, timeout=180)
+
+ # ... and re-org back to the first chain
+ self.move_tip(chain1_tip)
+ block = self.next_block(chain1_tip + 1)
+ self.sync_blocks([block], False, request_block=False)
+ block = self.next_block(chain1_tip + 2)
+ self.sync_blocks([block], True, timeout=180)
+
+ # Helper methods
+ ################
+
+ def add_transactions_to_block(self, block, tx_list):
+ [tx.rehash() for tx in tx_list]
+ block.vtx.extend(tx_list)
+
+ # this is a little handier to use than the version in blocktools.py
+ def create_tx(self, spend_tx, n, value, script=CScript([OP_TRUE])):
+ return create_transaction(spend_tx, n, b"", value, script)
+
+ # sign a transaction, using the key we know about
+ # this signs input 0 in tx, which is assumed to be spending output n in spend_tx
+ def sign_tx(self, tx, spend_tx, n):
+ scriptPubKey = bytearray(spend_tx.vout[n].scriptPubKey)
+ if (scriptPubKey[0] == OP_TRUE): # an anyone-can-spend
+ tx.vin[0].scriptSig = CScript()
+ return
+ (sighash, err) = SignatureHash(spend_tx.vout[n].scriptPubKey, tx, 0, SIGHASH_ALL)
+ tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))])
+
+ def create_and_sign_transaction(self, spend_tx, n, value, script=CScript([OP_TRUE])):
+ tx = self.create_tx(spend_tx, n, value, script)
+ self.sign_tx(tx, spend_tx, n)
+ tx.rehash()
+ return tx
+
+ def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True):
+ if self.tip is None:
+ base_block_hash = self.genesis_hash
+ block_time = int(time.time()) + 1
+ else:
+ base_block_hash = self.tip.sha256
+ block_time = self.tip.nTime + 1
+ # First create the coinbase
+ height = self.block_heights[base_block_hash] + 1
+ coinbase = create_coinbase(height, self.coinbase_pubkey)
+ coinbase.vout[0].nValue += additional_coinbase_value
+ coinbase.rehash()
+ if spend is None:
+ block = create_block(base_block_hash, coinbase, block_time)
+ else:
+ coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 # all but one satoshi to fees
+ coinbase.rehash()
+ block = create_block(base_block_hash, coinbase, block_time)
+ tx = create_transaction(spend.tx, spend.n, b"", 1, script) # spend 1 satoshi
+ self.sign_tx(tx, spend.tx, spend.n)
+ self.add_transactions_to_block(block, [tx])
+ block.hashMerkleRoot = block.calc_merkle_root()
+ if solve:
+ block.solve()
+ self.tip = block
+ self.block_heights[block.sha256] = height
+ assert number not in self.blocks
+ self.blocks[number] = block
+ return block
+
+ # save the current tip so it can be spent by a later block
+ def save_spendable_output(self):
+ self.log.debug("saving spendable output %s" % self.tip.vtx[0])
+ self.spendable_outputs.append(self.tip)
+
+ # get an output that we previously marked as spendable
+ def get_spendable_output(self):
+ self.log.debug("getting spendable output %s" % self.spendable_outputs[0].vtx[0])
+ return PreviousSpendableOutput(self.spendable_outputs.pop(0).vtx[0], 0)
+
+ # move the tip back to a previous block
+ def move_tip(self, number):
+ self.tip = self.blocks[number]
+
+ # adds transactions to the block and updates state
+ def update_block(self, block_number, new_transactions):
+ block = self.blocks[block_number]
+ self.add_transactions_to_block(block, new_transactions)
+ old_sha256 = block.sha256
+ block.hashMerkleRoot = block.calc_merkle_root()
+ block.solve()
+ # Update the internal state just like in next_block
+ self.tip = block
+ if block.sha256 != old_sha256:
+ self.block_heights[block.sha256] = self.block_heights[old_sha256]
+ del self.block_heights[old_sha256]
+ self.blocks[block_number] = block
+ return block
+
+ def reconnect_p2p(self):
+ """Add a P2P connection to the node.
+
+ The node gets disconnected several times in this test. This helper
+ method reconnects the p2p and restarts the network thread."""
+
+ network_thread_join()
+ self.nodes[0].disconnect_p2ps()
+ self.nodes[0].add_p2p_connection(P2PDataStore())
+ network_thread_start()
+ self.nodes[0].p2p.wait_for_verack()
+
+ def sync_blocks(self, blocks, success=True, reject_code=None, reject_reason=None, request_block=True, reconnect=False, timeout=60):
+ """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block.
+ Call with success = False if the tip shouldn't advance to the most recent block."""
+ self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_code=reject_code, reject_reason=reject_reason, request_block=request_block, timeout=timeout)
+ if reconnect:
+ self.reconnect_p2p()
if __name__ == '__main__':
FullBlockTest().main()
diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py
index 8e97df2361..32a6bd5d59 100755
--- a/test/functional/feature_fee_estimation.py
+++ b/test/functional/feature_fee_estimation.py
@@ -133,12 +133,12 @@ class EstimateFeeTest(BitcoinTestFramework):
which we will use to generate our transactions.
"""
self.add_nodes(3, extra_args=[["-maxorphantx=1000", "-whitelist=127.0.0.1"],
- ["-maxorphantx=1000"],
- ["-maxorphantx=1000"]])
+ ["-blockmaxweight=68000", "-maxorphantx=1000"],
+ ["-blockmaxweight=32000", "-maxorphantx=1000"]])
# Use node0 to mine blocks for input splitting
# Node1 mines small blocks but that are bigger than the expected transaction rate.
- # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes,
- # (17k is room enough for 110 or so transactions)
+ # NOTE: the CreateNewBlock code starts counting block weight at 4,000 weight,
+ # (68k weight is room enough for 120 or so transactions)
# Node2 is a stingy miner, that
# produces too small blocks (room for only 55 or so transactions)
diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index 227ae6cb36..3adde8dd73 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -124,7 +124,7 @@ class PruneTest(BitcoinTestFramework):
# Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
# Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
self.stop_node(1)
- self.start_node(1, extra_args=["-maxreceivebuffer=20000","-checkblocks=5", "-disablesafemode"])
+ self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5", "-disablesafemode"])
height = self.nodes[1].getblockcount()
self.log.info("Current block height: %d" % height)
@@ -147,7 +147,7 @@ class PruneTest(BitcoinTestFramework):
# Reboot node1 to clear those giant tx's from mempool
self.stop_node(1)
- self.start_node(1, extra_args=["-maxreceivebuffer=20000","-checkblocks=5", "-disablesafemode"])
+ self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5", "-disablesafemode"])
self.log.info("Generating new longer chain of 300 more blocks")
self.nodes[1].generate(300)
diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py
index 041e2b86e8..a24a2ec4f5 100644
--- a/test/functional/test_framework/util.py
+++ b/test/functional/test_framework/util.py
@@ -8,6 +8,7 @@ from base64 import b64encode
from binascii import hexlify, unhexlify
from decimal import Decimal, ROUND_DOWN
import hashlib
+import inspect
import json
import logging
import os
@@ -204,9 +205,9 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N
if attempts == float('inf') and timeout == float('inf'):
timeout = 60
attempt = 0
- timeout += time.time()
+ time_end = time.time() + timeout
- while attempt < attempts and time.time() < timeout:
+ while attempt < attempts and time.time() < time_end:
if lock:
with lock:
if predicate():
@@ -218,8 +219,12 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N
time.sleep(0.05)
# Print the cause of the timeout
- assert_greater_than(attempts, attempt)
- assert_greater_than(timeout, time.time())
+ predicate_source = inspect.getsourcelines(predicate)
+ logger.error("wait_until() failed. Predicate: {}".format(predicate_source))
+ if attempt >= attempts:
+ raise AssertionError("Predicate {} not true after {} attempts".format(predicate_source, attempts))
+ elif time.time() >= time_end:
+ raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout))
raise RuntimeError('Unreachable')
# RPC/P2P connection constants and functions