diff options
57 files changed, 595 insertions, 363 deletions
diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index b5a05e2629..ab13f73539 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -346,7 +346,7 @@ #define PACKAGE_NAME "Bitcoin Core" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 0.17.99" +#define PACKAGE_STRING "Bitcoin Core 0.18.99" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "bitcoin" @@ -355,7 +355,7 @@ #define PACKAGE_URL "https://bitcoincore.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "0.17.99" +#define PACKAGE_VERSION "0.18.99" /* Define to necessary symbol if this constant uses a non-standard name on your system. */ diff --git a/configure.ac b/configure.ac index d57e3b9196..5e1a5e14a1 100644 --- a/configure.ac +++ b/configure.ac @@ -194,6 +194,12 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=$enableval], [use_glibc_compat=no]) +AC_ARG_ENABLE([threadlocal], + [AS_HELP_STRING([--enable-threadlocal], + [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enabled if there is platform support and glibc-back-compat is not enabled)])], + [use_thread_local=$enableval], + [use_thread_local=auto]) + AC_ARG_ENABLE([asm], [AS_HELP_STRING([--disable-asm], [disable assembly routines (enabled by default)])], @@ -827,42 +833,49 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) -TEMP_LDFLAGS="$LDFLAGS" -LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" -AC_MSG_CHECKING([for thread_local support]) -AC_LINK_IFELSE([AC_LANG_SOURCE([ - #include <thread> - static thread_local int foo = 0; - static void run_thread() { foo++;} - int main(){ - for(int i = 0; i < 10; i++) { std::thread(run_thread).detach();} - return foo; - } - ])], - [ - case $host in - *mingw*) - # mingw32's implementation of thread_local has also been shown to behave - # erroneously under concurrent usage; see: - # https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 - AC_MSG_RESULT(no) - ;; - *darwin*) - # TODO enable thread_local on later versions of Darwin where it is - # supported (per https://stackoverflow.com/a/29929949) - AC_MSG_RESULT(no) - ;; - *) - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) - ;; - esac - ], - [ - AC_MSG_RESULT(no) - ] -) -LDFLAGS="$TEMP_LDFLAGS" +if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && test "x$use_glibc_compat" = xno; }; then + TEMP_LDFLAGS="$LDFLAGS" + LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" + AC_MSG_CHECKING([for thread_local support]) + AC_LINK_IFELSE([AC_LANG_SOURCE([ + #include <thread> + static thread_local int foo = 0; + static void run_thread() { foo++;} + int main(){ + for(int i = 0; i < 10; i++) { std::thread(run_thread).detach();} + return foo; + } + ])], + [ + case $host in + *mingw*) + # mingw32's implementation of thread_local has also been shown to behave + # erroneously under concurrent usage; see: + # https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 + AC_MSG_RESULT(no) + ;; + *darwin*) + # TODO enable thread_local on later versions of Darwin where it is + # supported (per https://stackoverflow.com/a/29929949) + AC_MSG_RESULT(no) + ;; + *freebsd*) + # FreeBSD's implementation of thread_local is also buggy (per + # https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) + AC_MSG_RESULT(no) + ;; + *) + AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) + AC_MSG_RESULT(yes) + ;; + esac + ], + [ + AC_MSG_RESULT(no) + ] + ) + LDFLAGS="$TEMP_LDFLAGS" +fi # Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index e51481e0bf..570d4906cd 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -7,20 +7,20 @@ import sys def setup(): global args, workdir - programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + programs = ['ruby', 'git', 'make', 'wget', 'curl'] if args.kvm: - programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker: + programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): dockers = ['docker.io', 'docker-ce'] for i in dockers: return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) if return_code == 0: break if return_code != 0: - print('Cannot find any way to install docker', file=sys.stderr) - exit(1) + print('Cannot find any way to install Docker.', file=sys.stderr) + sys.exit(1) else: - programs += ['lxc', 'debootstrap'] + programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian.sigs'): subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) @@ -41,7 +41,7 @@ def setup(): if args.is_bionic and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') - exit(0) + sys.exit(0) def build(): global args, workdir @@ -118,25 +118,41 @@ def sign(): def verify(): global args, workdir + rc = 0 os.chdir('gitian-builder') print('\nVerifying v'+args.version+' Linux\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']): + print('Verifying v'+args.version+' Linux FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Windows\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']): + print('Verifying v'+args.version+' Windows FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' MacOS\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']): + print('Verifying v'+args.version+' MacOS FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Signed Windows\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']): + print('Verifying v'+args.version+' Signed Windows FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Signed MacOS\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']): + print('Verifying v'+args.version+' Signed MacOS FAILED\n') + rc = 1 os.chdir(workdir) + return rc def main(): global args, workdir - parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') parser.add_argument('-u', '--url', dest='url', default='https://github.com/bitcoin/bitcoin', help='Specify the URL of the repository. Default is %(default)s') @@ -149,64 +165,70 @@ def main(): parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', help='GPG signer to sign each build assert file') - parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') + parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') args = parser.parse_args() workdir = os.getcwd() - args.linux = 'l' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) - if args.buildsign: - args.build=True - args.sign=True - if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - # Set environment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they + # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). + os.environ['USE_LXC'] = '' + os.environ['USE_VBOX'] = '' + os.environ['USE_DOCKER'] = '' if args.docker: os.environ['USE_DOCKER'] = '1' elif not args.kvm: os.environ['USE_LXC'] = '1' - if not 'GITIAN_HOST_IP' in os.environ.keys(): + if 'GITIAN_HOST_IP' not in os.environ.keys(): os.environ['GITIAN_HOST_IP'] = '10.0.3.1' - if not 'LXC_GUEST_IP' in os.environ.keys(): + if 'LXC_GUEST_IP' not in os.environ.keys(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' + if args.setup: + setup() + + if args.buildsign: + args.build = True + args.sign = True + + if not args.build and not args.sign and not args.verify: + sys.exit(0) + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + # Disable for MacOS if no SDK found if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + script_name = os.path.basename(sys.argv[0]) - # Signer and version shouldn't be empty - if args.signer == '': - print(script_name+': Missing signer.') + if not args.signer: + print(script_name+': Missing signer') print('Try '+script_name+' --help for more information') - exit(1) - if args.version == '': - print(script_name+': Missing version.') + sys.exit(1) + if not args.version: + print(script_name+': Missing version') print('Try '+script_name+' --help for more information') - exit(1) + sys.exit(1) # Add leading 'v' for tags if args.commit and args.pull: raise Exception('Cannot have both commit and pull') args.commit = ('' if args.commit else 'v') + args.version - if args.setup: - setup() - os.chdir('bitcoin') if args.pull: subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) @@ -219,6 +241,10 @@ def main(): subprocess.check_call(['git', 'checkout', args.commit]) os.chdir(workdir) + os.chdir('gitian-builder') + subprocess.check_call(['git', 'pull']) + os.chdir(workdir) + if args.build: build() @@ -226,7 +252,10 @@ def main(): sign() if args.verify: - verify() + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'pull']) + os.chdir(workdir) + sys.exit(verify()) if __name__ == '__main__': main() diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 6527deccf2..523386e393 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -109,18 +109,30 @@ def filtermultiport(ips): # Based on Greg Maxwell's seed_filter.py def filterbyasn(ips, max_per_asn, max_total): # Sift out ips by type - ips_ipv4 = [ip for ip in ips if ip['net'] == 'ipv4'] - ips_ipv6 = [ip for ip in ips if ip['net'] == 'ipv6'] + ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] - # Filter IPv4 by ASN + # Filter IPv46 by ASN result = [] asn_count = {} - for ip in ips_ipv4: + for ip in ips_ipv46: if len(result) == max_total: break try: - asn = int([x.to_text() for x in dns.resolver.query('.'.join(reversed(ip['ip'].split('.'))) + '.origin.asn.cymru.com', 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) + if ip['net'] == 'ipv4': + ipaddr = ip['ip'] + prefix = '.origin' + else: # http://www.team-cymru.com/IP-ASN-mapping.html + res = str() # 2001:4860:b002:23::68 + for nb in ip['ip'].split(':')[:4]: # pick the first 4 nibbles + for c in nb.zfill(4): # right padded with '0' + res += c + '.' # 2001 4860 b002 0023 + ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 + prefix = '.origin6' + + asn = int([x.to_text() for x in dns.resolver.query('.'.join( + reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', + 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) if asn not in asn_count: asn_count[asn] = 0 if asn_count[asn] == max_per_asn: @@ -130,10 +142,7 @@ def filterbyasn(ips, max_per_asn, max_total): except: sys.stderr.write('ERR: Could not resolve ASN for "' + ip['ip'] + '"\n') - # TODO: filter IPv6 by ASN - - # Add back non-IPv4 - result.extend(ips_ipv6) + # Add back Onions result.extend(ips_onion) return result diff --git a/depends/README.md b/depends/README.md index 11ef9d2b67..52ca618d9d 100644 --- a/depends/README.md +++ b/depends/README.md @@ -43,7 +43,7 @@ No other options are needed, the paths are automatically configured. Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 + sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch For linux ARM cross compilation: diff --git a/depends/packages.md b/depends/packages.md index 7c80362509..7d2bd4670d 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -14,8 +14,9 @@ Each package is required to define at least these variables: placeholder such as 1.0 can be used. $(package)_download_path: - Location of the upstream source, without the file-name. Usually http or - ftp. + Location of the upstream source, without the file-name. Usually http, https + or ftp. Secure transmission options like https should be preferred if + available. $(package)_file_name: The upstream source filename available at the download path. diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 3cd2e28858..6cdb79592b 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -1,6 +1,6 @@ package=bdb $(package)_version=4.8.30 -$(package)_download_path=http://download.oracle.com/berkeley-db +$(package)_download_path=https://download.oracle.com/berkeley-db $(package)_file_name=db-$($(package)_version).NC.tar.gz $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef $(package)_build_subdir=build_unix diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 12695db4b9..d0996b4534 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -1,6 +1,6 @@ package=fontconfig $(package)_version=2.12.1 -$(package)_download_path=http://www.freedesktop.org/software/fontconfig/release/ +$(package)_download_path=https://www.freedesktop.org/software/fontconfig/release/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3 $(package)_dependencies=freetype expat diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index 41e02e2030..a98e82ed16 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -1,6 +1,6 @@ package=freetype $(package)_version=2.7.1 -$(package)_download_path=http://download.savannah.gnu.org/releases/$(package) +$(package)_download_path=https://download.savannah.gnu.org/releases/$(package) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=3a3bb2c4e15ffb433f2032f50a5b5a92558206822e22bfe8cbe339af4aa82f88 diff --git a/depends/packages/libX11.mk b/depends/packages/libX11.mk index 298616bea4..a013da5192 100644 --- a/depends/packages/libX11.mk +++ b/depends/packages/libX11.mk @@ -1,6 +1,6 @@ package=libX11 $(package)_version=1.6.2 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=2aa027e837231d2eeea90f3a4afe19948a6eb4c8b2bec0241eba7dbc8106bd16 $(package)_dependencies=libxcb xtrans xextproto xproto diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 304494e3c5..ce42140689 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -1,6 +1,6 @@ package=libXau $(package)_version=1.0.8 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2 $(package)_dependencies=xproto diff --git a/depends/packages/libXext.mk b/depends/packages/libXext.mk index c0565dd672..458b967784 100644 --- a/depends/packages/libXext.mk +++ b/depends/packages/libXext.mk @@ -1,6 +1,6 @@ package=libXext $(package)_version=1.3.2 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=f829075bc646cdc085fa25d98d5885d83b1759ceb355933127c257e8e50432e0 $(package)_dependencies=xproto xextproto libX11 libXau diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 3f346d9728..3ddd5a7dd9 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -1,6 +1,6 @@ package=libxcb $(package)_version=1.10 -$(package)_download_path=http://xcb.freedesktop.org/dist +$(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5 $(package)_dependencies=xcb_proto libXau xproto diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 5ad2b580d2..fdbe22cda6 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,6 +1,6 @@ package=miniupnpc $(package)_version=2.0.20180203 -$(package)_download_path=http://miniupnp.free.fr/files +$(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7 diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index ccd72a99bd..a065256c1c 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -5,7 +5,7 @@ $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a $(package)_build_subdir=cctools $(package)_clang_version=3.7.1 -$(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_version) +$(package)_clang_download_path=https://llvm.org/releases/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9 diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk index cf694edb30..8243458ec8 100644 --- a/depends/packages/native_cdrkit.mk +++ b/depends/packages/native_cdrkit.mk @@ -1,6 +1,6 @@ package=native_cdrkit $(package)_version=1.1.11 -$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c +$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c $(package)_file_name=cdrkit-$($(package)_version).tar.bz2 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 $(package)_patches=cdrkit-deterministic.patch diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 0c7c958d62..44110394bd 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -1,6 +1,6 @@ package=xcb_proto $(package)_version=1.10 -$(package)_download_path=http://xcb.freedesktop.org/dist +$(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=xcb-proto-$($(package)_version).tar.bz2 $(package)_sha256_hash=7ef40ddd855b750bc597d2a435da21e55e502a0fefa85b274f2c922800baaf05 diff --git a/depends/packages/xextproto.mk b/depends/packages/xextproto.mk index 7065237bd5..157b76edf6 100644 --- a/depends/packages/xextproto.mk +++ b/depends/packages/xextproto.mk @@ -1,6 +1,6 @@ package=xextproto $(package)_version=7.3.0 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=f3f4b23ac8db9c3a9e0d8edb591713f3d70ef9c3b175970dd8823dfc92aa5bb0 diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 5328ec8481..23ad5ffa10 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -1,6 +1,6 @@ package=xproto $(package)_version=7.0.26 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f diff --git a/depends/packages/xtrans.mk b/depends/packages/xtrans.mk index c313b1f609..67d2d976c4 100644 --- a/depends/packages/xtrans.mk +++ b/depends/packages/xtrans.mk @@ -1,6 +1,6 @@ package=xtrans $(package)_version=1.3.4 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=054d4ee3efd52508c753e9f7bc655ef185a29bd2850dd9e2fc2ccc33544f583a $(package)_dependencies= diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk index 589490800f..1600b11a01 100644 --- a/depends/packages/zlib.mk +++ b/depends/packages/zlib.mk @@ -1,6 +1,6 @@ package=zlib $(package)_version=1.2.11 -$(package)_download_path=http://www.zlib.net +$(package)_download_path=https://www.zlib.net $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 diff --git a/doc/psbt.md b/doc/psbt.md index 9d85af0348..c411b31d5d 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -82,9 +82,10 @@ hardware implementations will typically implement multiple roles simultaneously. transactions. - **`decodepsbt`** is a diagnostic utility RPC which will show all information in a PSBT in human-readable form, as well as compute its eventual fee if known. -- **`analyzepsbt`** is a utility RPC that examines an RPC and reports the - next steps in the workflow if known, computes the fee of the resulting - transaction, and estimates the weight and feerate if possible. +- **`analyzepsbt`** is a utility RPC that examines a PSBT and reports the + current status of its inputs, the next step in the workflow if known, and if + possible, computes the fee of the resulting transaction and estimates the + final weight and feerate. ### Workflows diff --git a/doc/release-process.md b/doc/release-process.md index 9b796fa816..e9c6d5bf0d 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -27,10 +27,10 @@ Release Process * Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead. * Update `src/chainparams.cpp` chainTxData with statistics about the transaction count and rate. Use the output of the RPC `getchaintxstats`, see [this pull request](https://github.com/bitcoin/bitcoin/pull/12270) for an example. Reviewers can verify the results by running `getchaintxstats <window_block_count> <window_last_block_hash>` with the `window_block_count` and `window_last_block_hash` from your output. -* In `configure.ac` and `build_msvc/bitcoin_config.h` on _the master branch_: - - update `CLIENT_VERSION_MINOR` version -* In `configure.ac` and `build_msvc/bitcoin_config.h` on _a new release branch_ (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd972fca3dd4a33cfff90bf901b71a687e7)): - - update `CLIENT_VERSION_MINOR` version +* On both the master branch and the new release branch: + - update `CLIENT_VERSION_MINOR` in [`configure.ac`](../configure.ac) + - update `CLIENT_VERSION_MINOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) +* On the new release branch in [`configure.ac`](../configure.ac) and [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): - set `CLIENT_VERSION_REVISION` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 0a99ea3184..4016530dac 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -28,4 +28,13 @@ static void RollingBloom(benchmark::State& state) } } +static void RollingBloomReset(benchmark::State& state) +{ + CRollingBloomFilter filter(120000, 0.000001); + while (state.KeepRunning()) { + filter.reset(); + } +} + BENCHMARK(RollingBloom, 1500 * 1000); +BENCHMARK(RollingBloomReset, 20000); diff --git a/src/bloom.cpp b/src/bloom.cpp index 7732cee275..a061925089 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -14,6 +14,7 @@ #include <math.h> #include <stdlib.h> +#include <algorithm> #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 @@ -304,7 +305,5 @@ void CRollingBloomFilter::reset() nTweak = GetRand(std::numeric_limits<unsigned int>::max()); nEntriesThisGeneration = 0; nGeneration = 1; - for (std::vector<uint64_t>::iterator it = data.begin(); it != data.end(); it++) { - *it = 0; - } + std::fill(data.begin(), data.end(), 0); } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 63639fa3e0..d17667223b 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -139,15 +139,15 @@ struct HTTPPathHandler //! libevent event loop static struct event_base* eventBase = nullptr; //! HTTP server -struct evhttp* eventHTTP = nullptr; +static struct evhttp* eventHTTP = nullptr; //! List of subnets to allow RPC connections from static std::vector<CSubNet> rpc_allow_subnets; //! Work queue for handling longer requests off the event loop thread static WorkQueue<HTTPClosure>* workQueue = nullptr; //! Handlers for (sub)paths -std::vector<HTTPPathHandler> pathHandlers; +static std::vector<HTTPPathHandler> pathHandlers; //! Bound listening sockets -std::vector<evhttp_bound_socket *> boundSockets; +static std::vector<evhttp_bound_socket *> boundSockets; /** Check if a network address is allowed to access the HTTP server */ static bool ClientAllowed(const CNetAddr& netaddr) @@ -420,7 +420,7 @@ bool UpdateHTTPServerLogging(bool enable) { #endif } -std::thread threadHTTP; +static std::thread threadHTTP; static std::vector<std::thread> g_thread_http_workers; void StartHTTPServer() diff --git a/src/init.cpp b/src/init.cpp index 944200e24d..2bb270647a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -74,7 +74,7 @@ #include <zmq/zmqrpc.h> #endif -bool fFeeEstimatesInitialized = false; +static bool fFeeEstimatesInitialized = false; static const bool DEFAULT_PROXYRANDOMIZE = true; static const bool DEFAULT_REST_ENABLE = false; static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; @@ -1522,6 +1522,7 @@ bool AppInitMain(InitInterfaces& interfaces) // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! if (!LoadBlockIndex(chainparams)) { + if (ShutdownRequested()) break; strLoadError = _("Error loading block database"); break; } diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 6097d80931..a864f21f04 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -41,7 +41,7 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> { Optional<int> getHeight() override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); int height = ::ChainActive().Height(); if (height >= 0) { return height; @@ -50,7 +50,7 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } Optional<int> getBlockHeight(const uint256& hash) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = LookupBlockIndex(hash); if (block && ::ChainActive().Contains(block)) { return block->nHeight; @@ -65,34 +65,34 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } uint256 getBlockHash(int height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = ::ChainActive()[height]; assert(block != nullptr); return block->GetBlockHash(); } int64_t getBlockTime(int height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = ::ChainActive()[height]; assert(block != nullptr); return block->GetBlockTime(); } int64_t getBlockMedianTimePast(int height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = ::ChainActive()[height]; assert(block != nullptr); return block->GetMedianTimePast(); } bool haveBlockOnDisk(int height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = ::ChainActive()[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); if (block) { if (hash) *hash = block->GetBlockHash(); @@ -102,7 +102,7 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } Optional<int> findPruned(int start_height, Optional<int> stop_height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); if (::fPruneMode) { CBlockIndex* block = stop_height ? ::ChainActive()[*stop_height] : ::ChainActive().Tip(); while (block && block->nHeight >= start_height) { @@ -116,7 +116,7 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } Optional<int> findFork(const uint256& hash, Optional<int>* height) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); const CBlockIndex* block = LookupBlockIndex(hash); const CBlockIndex* fork = block ? ::ChainActive().FindFork(block) : nullptr; if (height) { @@ -133,12 +133,12 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } CBlockLocator getTipLocator() override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); return ::ChainActive().GetLocator(); } Optional<int> findLocatorFork(const CBlockLocator& locator) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { return fork->nHeight; } @@ -146,12 +146,12 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } bool checkFinalTx(const CTransaction& tx) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); return CheckFinalTx(tx); } bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) override { - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); return AcceptToMemoryPool(::mempool, state, tx, nullptr /* missing inputs */, nullptr /* txn replaced */, false /* bypass limits */, absurd_fee); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cb2b9f8c03..a3b865a69e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3288,23 +3288,22 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (m_enable_bip61) { connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); } - if (strstr(e.what(), "end of data")) - { + if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { + } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { + } else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { // Allow exceptions from non-canonical encoding LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { + } else if (strstr(e.what(), "Superfluous witness record")) { + // Allow exceptions from illegal witness encoding + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } else if (strstr(e.what(), "Unknown transaction optional data")) { + // Allow exceptions from unknown witness encoding + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } else { PrintExceptionContinue(&e, "ProcessMessages()"); } } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index abf9136eee..ff66df376b 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -370,9 +370,18 @@ void BitcoinGUI::createActions() connect(openAction, &QAction::triggered, this, &BitcoinGUI::openClicked); connect(m_open_wallet_action->menu(), &QMenu::aboutToShow, [this] { m_open_wallet_action->menu()->clear(); - for (std::string path : m_wallet_controller->getWalletsAvailableToOpen()) { + std::vector<std::string> available_wallets = m_wallet_controller->getWalletsAvailableToOpen(); + std::vector<std::string> wallets = m_node.listWalletDir(); + for (const auto& path : wallets) { QString name = path.empty() ? QString("["+tr("default wallet")+"]") : QString::fromStdString(path); QAction* action = m_open_wallet_action->menu()->addAction(name); + + if (std::find(available_wallets.begin(), available_wallets.end(), path) == available_wallets.end()) { + // This wallet is already loaded + action->setEnabled(false); + continue; + } + connect(action, &QAction::triggered, [this, name, path] { OpenWalletActivity* activity = m_wallet_controller->openWallet(path); @@ -400,6 +409,10 @@ void BitcoinGUI::createActions() assert(invoked); }); } + if (wallets.empty()) { + QAction* action = m_open_wallet_action->menu()->addAction(tr("No wallets available")); + action->setEnabled(false); + } }); connect(m_close_wallet_action, &QAction::triggered, [this] { m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); diff --git a/src/qt/forms/coincontroldialog.ui b/src/qt/forms/coincontroldialog.ui index bd7f3c5f56..5ce469ee96 100644 --- a/src/qt/forms/coincontroldialog.ui +++ b/src/qt/forms/coincontroldialog.ui @@ -467,12 +467,6 @@ </item> <item> <widget class="QDialogButtonBox" name="buttonBox"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Maximum" vsizetype="Fixed"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> <property name="orientation"> <enum>Qt::Horizontal</enum> </property> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 21209d4994..ab40e9962b 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -145,7 +145,7 @@ void TestGUI() } { auto locked_chain = wallet->chain().lock(); - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); WalletRescanReserver reserver(wallet.get()); reserver.reserve(); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index fd392b7cf7..a2b295df21 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -482,7 +482,7 @@ WalletModel::UnlockContext::~UnlockContext() } } -void WalletModel::UnlockContext::CopyFrom(const UnlockContext& rhs) +void WalletModel::UnlockContext::CopyFrom(UnlockContext&& rhs) { // Transfer context; old object no longer relocks wallet *this = rhs; diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index b123befbb4..54428aec08 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -194,15 +194,18 @@ public: bool isValid() const { return valid; } - // Copy operator and constructor transfer the context - UnlockContext(const UnlockContext& obj) { CopyFrom(obj); } - UnlockContext& operator=(const UnlockContext& rhs) { CopyFrom(rhs); return *this; } + // Copy constructor is disabled. + UnlockContext(const UnlockContext&) = delete; + // Move operator and constructor transfer the context + UnlockContext(UnlockContext&& obj) { CopyFrom(std::move(obj)); } + UnlockContext& operator=(UnlockContext&& rhs) { CopyFrom(std::move(rhs)); return *this; } private: WalletModel *wallet; bool valid; mutable bool relock; // mutable, as it can be set to false by copying - void CopyFrom(const UnlockContext& rhs); + UnlockContext& operator=(const UnlockContext&) = default; + void CopyFrom(UnlockContext&& rhs); }; UnlockContext requestUnlock(); diff --git a/src/random.cpp b/src/random.cpp index b08de60fbe..de26e6de1a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -237,6 +237,34 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { #endif } +/** Use repeated SHA512 to strengthen the randomness in seed32, and feed into hasher. */ +static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA512& hasher) noexcept +{ + CSHA512 inner_hasher; + inner_hasher.Write(seed, sizeof(seed)); + + // Hash loop + unsigned char buffer[64]; + int64_t stop = GetTimeMicros() + microseconds; + do { + for (int i = 0; i < 1000; ++i) { + inner_hasher.Finalize(buffer); + inner_hasher.Reset(); + inner_hasher.Write(buffer, sizeof(buffer)); + } + // Benchmark operation and feed it into outer hasher. + int64_t perf = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perf, sizeof(perf)); + } while (GetTimeMicros() < stop); + + // Produce output from inner state and feed it to outer hasher. + inner_hasher.Finalize(buffer); + hasher.Write(buffer, sizeof(buffer)); + // Try to clean up. + inner_hasher.Reset(); + memory_cleanse(buffer, sizeof(buffer)); +} + static void RandAddSeedPerfmon(CSHA512& hasher) { #ifdef WIN32 @@ -529,7 +557,23 @@ static void SeedSlow(CSHA512& hasher) noexcept SeedTimestamp(hasher); } -static void SeedSleep(CSHA512& hasher) +/** Extract entropy from rng, strengthen it, and feed it into hasher. */ +static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +{ + static std::atomic<int64_t> last_strengthen{0}; + int64_t last_time = last_strengthen.load(); + int64_t current_time = GetTimeMicros(); + if (current_time > last_time + 60000000) { // Only run once a minute + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen it for 10ms (100ms on first run), and feed it into hasher. + Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); + last_strengthen = current_time; + } +} + +static void SeedSleep(CSHA512& hasher, RNGState& rng) { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -545,9 +589,12 @@ static void SeedSleep(CSHA512& hasher) // Windows performance monitor data (once every 10 minutes) RandAddSeedPerfmon(hasher); + + // Strengthen every minute + SeedStrengthen(hasher, rng); } -static void SeedStartup(CSHA512& hasher) noexcept +static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept { #ifdef WIN32 RAND_screen(); @@ -561,6 +608,9 @@ static void SeedStartup(CSHA512& hasher) noexcept // Windows performance monitor data. RandAddSeedPerfmon(hasher); + + // Strengthen + SeedStrengthen(hasher, rng); } enum class RNGLevel { @@ -585,7 +635,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedSlow(hasher); break; case RNGLevel::SLEEP: - SeedSleep(hasher); + SeedSleep(hasher, rng); break; } @@ -593,7 +643,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) if (!rng.MixExtract(out, num, std::move(hasher), false)) { // On the first invocation, also seed with SeedStartup(). CSHA512 startup_hasher; - SeedStartup(startup_hasher); + SeedStartup(startup_hasher, rng); rng.MixExtract(out, num, std::move(startup_hasher), true); } diff --git a/src/random.h b/src/random.h index 2f9c0f5a36..75d037738d 100644 --- a/src/random.h +++ b/src/random.h @@ -43,6 +43,7 @@ * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: * - A high-precision timestamp before and after sleeping 1ms. * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. + - - Once every minute, strengthen the entropy for 10 ms using repeated SHA512. * These just exploit the fact the system is idle to improve the quality of the RNG * slightly. * @@ -51,6 +52,7 @@ * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. * - (On Windows) Through OpenSSL, the screen contents. + * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and * (up to) the first 32 bytes of H are produced as output, while the last 32 bytes diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 7008a83143..72a2919712 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -204,7 +204,7 @@ UniValue deriveaddresses(const JSONRPCRequest& request) }, RPCExamples{ "First three native segwit receive addresses\n" + - HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#trd0mf0l\" \"[0,2]\"") + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu\" \"[0,2]\"") }}.ToString() ); } diff --git a/src/sync.h b/src/sync.h index 2667fb52f9..bdbdde1a2a 100644 --- a/src/sync.h +++ b/src/sync.h @@ -304,4 +304,18 @@ public: } }; +// Utility class for indicating to compiler thread analysis that a mutex is +// locked (when it couldn't be determined otherwise). +struct SCOPED_LOCKABLE LockAssertion +{ + template <typename Mutex> + explicit LockAssertion(Mutex& mutex) EXCLUSIVE_LOCK_FUNCTION(mutex) + { +#ifdef DEBUG_LOCKORDER + AssertLockHeld(mutex); +#endif + } + ~LockAssertion() UNLOCK_FUNCTION() {} +}; + #endif // BITCOIN_SYNC_H diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 38d91b6647..0ca3a17974 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -271,12 +271,12 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // Before MedianTimePast of the chain has crossed nStartTime, the bit // should not be set. CBlockIndex *lastBlock = nullptr; - lastBlock = firstChain.Mine(2016, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + lastBlock = firstChain.Mine(mainnetParams.nMinerConfirmationWindow, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit), 0); - // Mine 2011 more blocks at the old time, and check that CBV isn't setting the bit yet. - for (int i=1; i<2012; i++) { - lastBlock = firstChain.Mine(2016+i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + // Mine more blocks (4 less than the adjustment period) at the old time, and check that CBV isn't setting the bit yet. + for (uint32_t i = 1; i < mainnetParams.nMinerConfirmationWindow - 4; i++) { + lastBlock = firstChain.Mine(mainnetParams.nMinerConfirmationWindow + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); // This works because VERSIONBITS_LAST_OLD_BLOCK_VERSION happens // to be 4, and the bit we're testing happens to be bit 28. BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit), 0); @@ -284,13 +284,13 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // Now mine 5 more blocks at the start time -- MTP should not have passed yet, so // CBV should still not yet set the bit. nTime = nStartTime; - for (int i=2012; i<=2016; i++) { - lastBlock = firstChain.Mine(2016+i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + for (uint32_t i = mainnetParams.nMinerConfirmationWindow - 4; i <= mainnetParams.nMinerConfirmationWindow; i++) { + lastBlock = firstChain.Mine(mainnetParams.nMinerConfirmationWindow + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit), 0); } // Advance to the next period and transition to STARTED, - lastBlock = firstChain.Mine(6048, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + lastBlock = firstChain.Mine(mainnetParams.nMinerConfirmationWindow * 3, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); // so ComputeBlockVersion should now set the bit, BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit)) != 0); // and should also be using the VERSIONBITS_TOP_BITS. @@ -298,8 +298,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // Check that ComputeBlockVersion will set the bit until nTimeout nTime += 600; - int blocksToMine = 4032; // test blocks for up to 2 time periods - int nHeight = 6048; + uint32_t blocksToMine = mainnetParams.nMinerConfirmationWindow * 2; // test blocks for up to 2 time periods + uint32_t nHeight = mainnetParams.nMinerConfirmationWindow * 3; // These blocks are all before nTimeout is reached. while (nTime < nTimeout && blocksToMine > 0) { lastBlock = firstChain.Mine(nHeight+1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); @@ -313,7 +313,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) nTime = nTimeout; // FAILED is only triggered at the end of a period, so CBV should be setting // the bit until the period transition. - for (int i=0; i<2015; i++) { + for (uint32_t i = 0; i < mainnetParams.nMinerConfirmationWindow - 1; i++) { lastBlock = firstChain.Mine(nHeight+1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit)) != 0); nHeight += 1; @@ -329,20 +329,20 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // Mine one period worth of blocks, and check that the bit will be on for the // next period. - lastBlock = secondChain.Mine(2016, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + lastBlock = secondChain.Mine(mainnetParams.nMinerConfirmationWindow, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit)) != 0); // Mine another period worth of blocks, signaling the new bit. - lastBlock = secondChain.Mine(4032, nTime, VERSIONBITS_TOP_BITS | (1<<bit)).Tip(); + lastBlock = secondChain.Mine(mainnetParams.nMinerConfirmationWindow * 2, nTime, VERSIONBITS_TOP_BITS | (1<<bit)).Tip(); // After one period of setting the bit on each block, it should have locked in. // We keep setting the bit for one more period though, until activation. BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit)) != 0); // Now check that we keep mining the block until the end of this period, and // then stop at the beginning of the next period. - lastBlock = secondChain.Mine(6047, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit)) != 0); - lastBlock = secondChain.Mine(6048, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + lastBlock = secondChain.Mine((mainnetParams.nMinerConfirmationWindow * 3) - 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); + BOOST_CHECK((ComputeBlockVersion(lastBlock, mainnetParams) & (1 << bit)) != 0); + lastBlock = secondChain.Mine(mainnetParams.nMinerConfirmationWindow * 3, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, mainnetParams) & (1<<bit), 0); // Finally, verify that after a soft fork has activated, CBV no longer uses diff --git a/src/threadsafety.h b/src/threadsafety.h index 33acddc65c..47e6b2ea38 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -54,15 +54,4 @@ #define ASSERT_EXCLUSIVE_LOCK(...) #endif // __GNUC__ -// Utility class for indicating to compiler thread analysis that a mutex is -// locked (when it couldn't be determined otherwise). -struct SCOPED_LOCKABLE LockAnnotation -{ - template <typename Mutex> - explicit LockAnnotation(Mutex& mutex) EXCLUSIVE_LOCK_FUNCTION(mutex) - { - } - ~LockAnnotation() UNLOCK_FUNCTION() {} -}; - #endif // BITCOIN_THREADSAFETY_H diff --git a/src/txdb.cpp b/src/txdb.cpp index 8447352c54..494b87ad48 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -255,6 +255,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); + if (ShutdownRequested()) return false; std::pair<char, uint256> key; if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { CDiskBlockIndex diskindex; diff --git a/src/validation.cpp b/src/validation.cpp index b2d84f4c95..436c62261b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3861,6 +3861,7 @@ bool CChainState::LoadBlockIndex(const Consensus::Params& consensus_params, CBlo sort(vSortedByHeight.begin(), vSortedByHeight.end()); for (const std::pair<int, CBlockIndex*>& item : vSortedByHeight) { + if (ShutdownRequested()) return false; CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); @@ -4306,6 +4307,7 @@ bool RewindBlockIndex(const CChainParams& params) { return false; } + LOCK(cs_main); if (::ChainActive().Tip() != nullptr) { // FlushStateToDisk can possibly read ::ChainActive(). Be conservative // and skip it here, we're about to -reindex-chainstate anyway, so diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 546e3d54eb..8633d8701b 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -607,7 +607,9 @@ void BerkeleyBatch::Flush() if (fReadOnly) nMinutes = 1; - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault + env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + } } void BerkeleyDatabase::IncrementUpdateCounter() diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 3861b5524f..ee1b792f9b 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -116,7 +116,8 @@ UniValue importprivkey(const JSONRPCRequest& request) "\nAdds a private key (as returned by dumpprivkey) to your wallet. Requires a new wallet backup.\n" "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" - "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n", + "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, /* default */ "current label if address exists, otherwise \"\"", "An optional label"}, @@ -220,7 +221,8 @@ UniValue abortrescan(const JSONRPCRequest& request) if (request.fHelp || request.params.size() > 0) throw std::runtime_error( RPCHelpMan{"abortrescan", - "\nStops current wallet rescan triggered by an RPC call, e.g. by an importprivkey call.\n", + "\nStops current wallet rescan triggered by an RPC call, e.g. by an importprivkey call.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", {}, RPCResults{}, RPCExamples{ @@ -289,8 +291,10 @@ UniValue importaddress(const JSONRPCRequest& request) "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" "If you have the full public key, you should call importpubkey instead of this.\n" + "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" - "as change, and not show up in many RPCs.\n", + "as change, and not show up in many RPCs.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, /* default */ "\"\"", "An optional label"}, @@ -486,8 +490,10 @@ UniValue importpubkey(const JSONRPCRequest& request) throw std::runtime_error( RPCHelpMan{"importpubkey", "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" + "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" - "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n", + "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, /* default */ "\"\"", "An optional label"}, @@ -568,7 +574,8 @@ UniValue importwallet(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( RPCHelpMan{"importwallet", - "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n", + "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -1266,55 +1273,17 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con // All good, time to import pwallet->MarkDirty(); - for (const auto& entry : import_data.import_scripts) { - if (!pwallet->HaveCScript(CScriptID(entry)) && !pwallet->AddCScript(entry)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet"); - } - } - for (const auto& entry : privkey_map) { - const CKey& key = entry.second; - CPubKey pubkey = key.GetPubKey(); - const CKeyID& id = entry.first; - assert(key.VerifyPubKey(pubkey)); - pwallet->mapKeyMetadata[id].nCreateTime = timestamp; - // If the private key is not present in the wallet, insert it. - if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); - } - pwallet->UpdateTimeFirstKey(timestamp); + if (!pwallet->ImportScripts(import_data.import_scripts)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet"); } - for (const auto& entry : import_data.key_origins) { - pwallet->AddKeyOrigin(entry.second.first, entry.second.second); + if (!pwallet->ImportPrivKeys(privkey_map, timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } - for (const CKeyID& id : ordered_pubkeys) { - auto entry = pubkey_map.find(id); - if (entry == pubkey_map.end()) { - continue; - } - const CPubKey& pubkey = entry->second; - CPubKey temp; - if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); - } - pwallet->mapKeyMetadata[id].nCreateTime = timestamp; - - // Add to keypool only works with pubkeys - if (add_keypool) { - pwallet->AddKeypoolPubkey(pubkey, internal); - } + if (!pwallet->ImportPubKeys(ordered_pubkeys, pubkey_map, import_data.key_origins, add_keypool, internal, timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } - - for (const CScript& script : script_pub_keys) { - if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated - if (!pwallet->AddWatchOnly(script, timestamp)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); - } - } - CTxDestination dest; - ExtractDestination(script, dest); - if (!internal && IsValidDestination(dest)) { - pwallet->SetAddressBook(dest, label, "receive"); - } + if (!pwallet->ImportScriptPubKeys(label, script_pub_keys, have_solving_data, internal, timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } result.pushKV("success", UniValue(true)); @@ -1359,7 +1328,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) "If an address/script is imported without all of the private keys required to spend from that address, it will be watchonly. The 'watchonly' option must be set to true in this case or a warning will be returned.\n" "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" - "may report that the imported keys, addresses or scripts exists but related transactions are still missing.\n", + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 96a74a338a..71d62a47dc 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3452,7 +3452,8 @@ UniValue rescanblockchain(const JSONRPCRequest& request) if (request.fHelp || request.params.size() > 2) { throw std::runtime_error( RPCHelpMan{"rescanblockchain", - "\nRescan the local blockchain for wallet related transactions.\n", + "\nRescan the local blockchain for wallet related transactions.\n" + "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"start_height", RPCArg::Type::NUM, /* default */ "0", "block height where the rescan should start"}, {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 62630c011a..922bb0fe65 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -44,7 +44,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) auto chain = interfaces::MakeChain(); auto locked_chain = chain->lock(); - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); // Verify ScanForWalletTransactions accommodates a null start block. { @@ -123,7 +123,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) auto chain = interfaces::MakeChain(); auto locked_chain = chain->lock(); - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); // Prune the older block file. PruneOneBlockFile(oldTip->GetBlockPos().nFile); @@ -190,7 +190,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) auto chain = interfaces::MakeChain(); auto locked_chain = chain->lock(); - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); std::string backup_file = (SetDataDir("importwallet_rescan") / "wallet.backup").string(); @@ -248,7 +248,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); - LockAnnotation lock(::cs_main); + LockAssertion lock(::cs_main); LOCK(wallet.cs_wallet); wtx.hashBlock = ::ChainActive().Tip()->GetBlockHash(); @@ -272,8 +272,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - LockAnnotation lock(::cs_main); auto locked_chain = wallet.chain().lock(); + LockAssertion lock(::cs_main); auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 260617c666..e86147b5d0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -320,7 +320,7 @@ bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const C secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } - UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); return true; } @@ -362,12 +362,6 @@ void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& m_script_metadata[script_id] = meta; } -// Writes a keymetadata for a public key. overwrite specifies whether to overwrite an existing metadata for that key if there exists one. -bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite) -{ - return WalletBatch(*database).WriteKeyMetadata(meta, pubkey, overwrite); -} - void CWallet::UpgradeKeyMetadata() { AssertLockHeld(cs_wallet); @@ -376,7 +370,6 @@ void CWallet::UpgradeKeyMetadata() } std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database); - size_t cnt = 0; for (auto& meta_pair : mapKeyMetadata) { CKeyMetadata& meta = meta_pair.second; if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin @@ -399,10 +392,6 @@ void CWallet::UpgradeKeyMetadata() CPubKey pubkey; if (GetPubKey(meta_pair.first, pubkey)) { batch->WriteKeyMetadata(meta, pubkey, true); - if (++cnt % 1000 == 0) { - // avoid creating overlarge in-memory batches in case the wallet contains large amounts of keys - batch.reset(new WalletBatch(*database)); - } } } } @@ -433,10 +422,16 @@ void CWallet::UpdateTimeFirstKey(int64_t nCreateTime) bool CWallet::AddCScript(const CScript& redeemScript) { + WalletBatch batch(*database); + return AddCScriptWithDB(batch, redeemScript); +} + +bool CWallet::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript) +{ if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - if (WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript)) { - UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); return true; } return false; @@ -457,20 +452,32 @@ bool CWallet::LoadCScript(const CScript& redeemScript) return CCryptoKeyStore::AddCScript(redeemScript); } -bool CWallet::AddWatchOnly(const CScript& dest) +bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) { if (!CCryptoKeyStore::AddWatchOnly(dest)) return false; const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - if (WalletBatch(*database).WriteWatchOnly(dest, meta)) { - UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + if (batch.WriteWatchOnly(dest, meta)) { + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); return true; } return false; } +bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) +{ + m_script_metadata[CScriptID(dest)].nCreateTime = create_time; + return AddWatchOnlyWithDB(batch, dest); +} + +bool CWallet::AddWatchOnly(const CScript& dest) +{ + WalletBatch batch(*database); + return AddWatchOnlyWithDB(batch, dest); +} + bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) { m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; @@ -1543,9 +1550,15 @@ void CWallet::SetWalletFlag(uint64_t flags) void CWallet::UnsetWalletFlag(uint64_t flag) { + WalletBatch batch(*database); + UnsetWalletFlagWithDB(batch, flag); +} + +void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag) +{ LOCK(cs_wallet); m_wallet_flags &= ~flag; - if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) + if (!batch.WriteWalletFlags(m_wallet_flags)) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } @@ -1606,6 +1619,80 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> return true; } +bool CWallet::ImportScripts(const std::set<CScript> scripts) +{ + WalletBatch batch(*database); + for (const auto& entry : scripts) { + if (!HaveCScript(CScriptID(entry)) && !AddCScriptWithDB(batch, entry)) { + return false; + } + } + return true; +} + +bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) +{ + WalletBatch batch(*database); + for (const auto& entry : privkey_map) { + const CKey& key = entry.second; + CPubKey pubkey = key.GetPubKey(); + const CKeyID& id = entry.first; + assert(key.VerifyPubKey(pubkey)); + mapKeyMetadata[id].nCreateTime = timestamp; + // If the private key is not present in the wallet, insert it. + if (!HaveKey(id) && !AddKeyPubKeyWithDB(batch, key, pubkey)) { + return false; + } + UpdateTimeFirstKey(timestamp); + } + return true; +} + +bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) +{ + WalletBatch batch(*database); + for (const auto& entry : key_origins) { + AddKeyOriginWithDB(batch, entry.second.first, entry.second.second); + } + for (const CKeyID& id : ordered_pubkeys) { + auto entry = pubkey_map.find(id); + if (entry == pubkey_map.end()) { + continue; + } + const CPubKey& pubkey = entry->second; + CPubKey temp; + if (!GetPubKey(id, temp) && !AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) { + return false; + } + mapKeyMetadata[id].nCreateTime = timestamp; + + // Add to keypool only works with pubkeys + if (add_keypool) { + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + NotifyCanGetAddressesChanged(); + } + } + return true; +} + +bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool internal, const int64_t timestamp) +{ + WalletBatch batch(*database); + for (const CScript& script : script_pub_keys) { + if (!have_solving_data || !::IsMine(*this, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated + if (!AddWatchOnlyWithDB(batch, script, timestamp)) { + return false; + } + } + CTxDestination dest; + ExtractDestination(script, dest); + if (!internal && IsValidDestination(dest)) { + SetAddressBookWithDB(batch, dest, label, "receive"); + } + } + return true; +} + int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig) { std::vector<CTxOut> txouts; @@ -3149,8 +3236,7 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) return DBErrors::LOAD_OK; } - -bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose) +bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose) { bool fUpdated = false; { @@ -3163,9 +3249,15 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s } NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); - if (!strPurpose.empty() && !WalletBatch(*database).WritePurpose(EncodeDestination(address), strPurpose)) + if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; - return WalletBatch(*database).WriteName(EncodeDestination(address), strName); + return batch.WriteName(EncodeDestination(address), strName); +} + +bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose) +{ + WalletBatch batch(*database); + return SetAddressBookWithDB(batch, address, strName, strPurpose); } bool CWallet::DelAddressBook(const CTxDestination& address) @@ -3315,13 +3407,6 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) return true; } -void CWallet::AddKeypoolPubkey(const CPubKey& pubkey, const bool internal) -{ - WalletBatch batch(*database); - AddKeypoolPubkeyWithDB(pubkey, internal, batch); - NotifyCanGetAddressesChanged(); -} - void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) { LOCK(cs_wallet); @@ -4443,12 +4528,12 @@ bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const return true; } -bool CWallet::AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info) +bool CWallet::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) { LOCK(cs_wallet); std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; mapKeyMetadata[pubkey.GetID()].has_key_origin = true; mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); - return WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); + return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5e3bbe2f37..87aff09039 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -775,6 +775,26 @@ private: * nTimeFirstKey more intelligently for more efficient rescans. */ bool AddWatchOnly(const CScript& dest) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); + + //! Adds a key to the store, and saves it to disk. + bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Adds a watch-only address to the store, and saves it to disk. + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); + + bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); + + //! Adds a script to the store and saves it to disk + bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); + + //! Unsets a wallet flag and saves it to disk + void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag); /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -833,8 +853,6 @@ public: // Map from Script ID to key metadata (for watch-only keys). std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); - bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, bool overwrite); - typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; @@ -930,7 +948,6 @@ public: CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a key to the store, without saving it to disk (used by LoadWallet) bool LoadKey(const CKey& key, const CPubKey &pubkey) { return CCryptoKeyStore::AddKeyPubKey(key, pubkey); } //! Load metadata (used by LoadWallet) @@ -1049,6 +1066,11 @@ public: bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig = false) const; bool DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig = false) const; + bool ImportScripts(const std::set<CScript> scripts) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; bool m_spend_zero_conf_change{DEFAULT_SPEND_ZEROCONF_CHANGE}; @@ -1070,8 +1092,6 @@ public: bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); - void AddKeypoolPubkey(const CPubKey& pubkey, const bool internal); - void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); /** * Reserves a key from the keypool and sets nIndex to its index @@ -1288,9 +1308,6 @@ public: /** Implement lookup of key origin information through wallet key metadata. */ bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; - - /** Add a KeyOriginInfo to the wallet */ - bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info); }; /** diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 0532a55ff5..d4a3bba97a 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -143,9 +143,11 @@ public: }; /** Access to the wallet database. - * This represents a single transaction at the - * database. It will be committed when the object goes out of scope. - * Optionally (on by default) it will flush to disk as well. + * Opens the database and provides read and write access to it. Each read and write is its own transaction. + * Multiple operation transactions can be started using TxnBegin() and committed using TxnCommit() + * Otherwise the transaction will be committed when the object goes out of scope. + * Optionally (on by default) it will flush to disk on close. + * Every 1000 writes will automatically trigger a flush to disk. */ class WalletBatch { @@ -157,6 +159,9 @@ private: return false; } m_database.IncrementUpdateCounter(); + if (m_database.nUpdateCounter % 1000 == 0) { + m_batch.Flush(); + } return true; } @@ -167,6 +172,9 @@ private: return false; } m_database.IncrementUpdateCounter(); + if (m_database.nUpdateCounter % 1000 == 0) { + m_batch.Flush(); + } return true; } diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 5c1b086703..c6132a2686 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -59,7 +59,7 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa try { bool first_run; load_wallet_ret = wallet_instance->LoadWallet(first_run); - } catch (const std::runtime_error) { + } catch (const std::runtime_error&) { fprintf(stderr, "Error loading %s. Is wallet being used by another process?\n", name.c_str()); return nullptr; } diff --git a/src/warnings.cpp b/src/warnings.cpp index 1c6ba13f60..8ac9ee09db 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -8,10 +8,10 @@ #include <util/system.h> #include <warnings.h> -CCriticalSection cs_warnings; -std::string strMiscWarning GUARDED_BY(cs_warnings); -bool fLargeWorkForkFound GUARDED_BY(cs_warnings) = false; -bool fLargeWorkInvalidChainFound GUARDED_BY(cs_warnings) = false; +static RecursiveMutex cs_warnings; +static std::string strMiscWarning GUARDED_BY(cs_warnings); +static bool fLargeWorkForkFound GUARDED_BY(cs_warnings) = false; +static bool fLargeWorkInvalidChainFound GUARDED_BY(cs_warnings) = false; void SetMiscWarning(const std::string& strWarning) { diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 000c30646a..df0f9fa365 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -2038,9 +2038,9 @@ class SegWitTest(BitcoinTestFramework): # TODO: test p2sh sigop counting def test_superfluous_witness(self): - # Serialization of tx that puts witness flag to 1 always + # Serialization of tx that puts witness flag to 3 always def serialize_with_bogus_witness(tx): - flags = 1 + flags = 3 r = b"" r += struct.pack("<i", tx.nVersion) if flags: @@ -2059,9 +2059,29 @@ class SegWitTest(BitcoinTestFramework): r += struct.pack("<I", tx.nLockTime) return r - raw = self.nodes[0].createrawtransaction([{"txid":"00"*32, "vout":0}], {self.nodes[0].getnewaddress():1}) + class msg_bogus_tx(msg_tx): + def serialize(self): + return serialize_with_bogus_witness(self.tx) + + self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) + self.nodes[0].generate(1) + unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) + + raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) tx = FromHex(CTransaction(), raw) assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, serialize_with_bogus_witness(tx).hex()) + with self.nodes[0].assert_debug_log(['Superfluous witness record']): + self.nodes[0].p2p.send_message(msg_bogus_tx(tx)) + self.nodes[0].p2p.sync_with_ping() + raw = self.nodes[0].signrawtransactionwithwallet(raw) + assert raw['complete'] + raw = raw['hex'] + tx = FromHex(CTransaction(), raw) + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, serialize_with_bogus_witness(tx).hex()) + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + self.nodes[0].p2p.send_message(msg_bogus_tx(tx)) + self.nodes[0].p2p.sync_with_ping() + if __name__ == '__main__': SegWitTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 6e71817dc3..4338675270 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -42,7 +42,11 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]] + self.extra_args = [ + ["-txindex"], + ["-txindex"], + ["-txindex"], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -361,30 +365,30 @@ class RawTransactionsTest(BitcoinTestFramework): # getrawtransaction tests # 1. valid parameters - only supply txid - txHash = rawTx["hash"] - assert_equal(self.nodes[0].getrawtransaction(txHash), rawTxSigned['hex']) + txId = rawTx["txid"] + assert_equal(self.nodes[0].getrawtransaction(txId), rawTxSigned['hex']) # 2. valid parameters - supply txid and 0 for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txHash, 0), rawTxSigned['hex']) + assert_equal(self.nodes[0].getrawtransaction(txId, 0), rawTxSigned['hex']) # 3. valid parameters - supply txid and False for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txHash, False), rawTxSigned['hex']) + assert_equal(self.nodes[0].getrawtransaction(txId, False), rawTxSigned['hex']) # 4. valid parameters - supply txid and 1 for verbose. # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. - assert_equal(self.nodes[0].getrawtransaction(txHash, 1)["hex"], rawTxSigned['hex']) + assert_equal(self.nodes[0].getrawtransaction(txId, 1)["hex"], rawTxSigned['hex']) # 5. valid parameters - supply txid and True for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) + assert_equal(self.nodes[0].getrawtransaction(txId, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, []) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } @@ -438,7 +442,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx = self.nodes[2].createrawtransaction(inputs, outputs) rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) assert_equal(rawTxSigned['complete'], True) - # 1000 sat fee, ~200 b transaction, fee rate should land around 5 sat/b = 0.00005000 BTC/kB + # 1000 sat fee, ~100 b transaction, fee rate should land around 10 sat/b = 0.00010000 BTC/kB # Thus, testmempoolaccept should reject testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] assert_equal(testres['allowed'], False) @@ -446,9 +450,9 @@ class RawTransactionsTest(BitcoinTestFramework): # and sendrawtransaction should throw assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) # And below calls should both succeed - testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.00007000')[0] + testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.00070000')[0] assert_equal(testres['allowed'], True) - self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.00007000') + self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.00070000') if __name__ == '__main__': diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 2187bf5f5f..aa674f9ebe 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -29,7 +29,6 @@ from .util import ( disconnect_nodes, get_datadir_path, initialize_datadir, - p2p_port, sync_blocks, sync_mempools, ) @@ -468,35 +467,23 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. - Create a cache of a 199-block-long chain (with wallet) for MAX_NODES + Create a cache of a 199-block-long chain Afterward, create num_nodes copies from the cache.""" + CACHE_NODE_ID = 0 # Use node 0 to create the cache for all other nodes + cache_node_dir = get_datadir_path(self.options.cachedir, CACHE_NODE_ID) assert self.num_nodes <= MAX_NODES - create_cache = False - for i in range(MAX_NODES): - if not os.path.isdir(get_datadir_path(self.options.cachedir, i)): - create_cache = True - break - - if create_cache: - self.log.debug("Creating data directories from cached datadir") - - # find and delete old cache directories if any exist - for i in range(MAX_NODES): - if os.path.isdir(get_datadir_path(self.options.cachedir, i)): - shutil.rmtree(get_datadir_path(self.options.cachedir, i)) - - # Create cache directories, run bitcoinds: - for i in range(MAX_NODES): - datadir = initialize_datadir(self.options.cachedir, i) - args = [self.options.bitcoind, "-datadir=" + datadir, '-disablewallet'] - if i > 0: - args.append("-connect=127.0.0.1:" + str(p2p_port(0))) - self.nodes.append(TestNode( - i, - get_datadir_path(self.options.cachedir, i), + + if not os.path.isdir(cache_node_dir): + self.log.debug("Creating cache directory {}".format(cache_node_dir)) + + initialize_datadir(self.options.cachedir, CACHE_NODE_ID) + self.nodes.append( + TestNode( + CACHE_NODE_ID, + cache_node_dir, extra_conf=["bind=127.0.0.1"], - extra_args=[], + extra_args=['-disablewallet'], rpchost=None, timewait=self.rpc_timeout, bitcoind=self.options.bitcoind, @@ -504,12 +491,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): coverage_dir=None, cwd=self.options.tmpdir, )) - self.nodes[i].args = args - self.start_node(i) + self.start_node(CACHE_NODE_ID) # Wait for RPC connections to be ready - for node in self.nodes: - node.wait_for_rpc_connection() + self.nodes[CACHE_NODE_ID].wait_for_rpc_connection() # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. @@ -518,29 +503,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). for i in range(8): - self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address) - self.sync_blocks() + self.nodes[CACHE_NODE_ID].generatetoaddress( + nblocks=25 if i != 7 else 24, + address=TestNode.PRIV_KEYS[i % 4].address, + ) - for n in self.nodes: - assert_equal(n.getblockchaininfo()["blocks"], 199) + assert_equal(self.nodes[CACHE_NODE_ID].getblockchaininfo()["blocks"], 199) - # Shut them down, and clean up cache directories: + # Shut it down, and clean up cache directories: self.stop_nodes() self.nodes = [] - def cache_path(n, *paths): - return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths) + def cache_path(*paths): + return os.path.join(cache_node_dir, "regtest", *paths) - for i in range(MAX_NODES): - os.rmdir(cache_path(i, 'wallets')) # Remove empty wallets dir - for entry in os.listdir(cache_path(i)): - if entry not in ['chainstate', 'blocks']: - os.remove(cache_path(i, entry)) + os.rmdir(cache_path('wallets')) # Remove empty wallets dir + for entry in os.listdir(cache_path()): + if entry not in ['chainstate', 'blocks']: # Only keep chainstate and blocks folder + os.remove(cache_path(entry)) for i in range(self.num_nodes): - from_dir = get_datadir_path(self.options.cachedir, i) + self.log.debug("Copy cache directory {} to node {}".format(cache_node_dir, i)) to_dir = get_datadir_path(self.options.tmpdir, i) - shutil.copytree(from_dir, to_dir) + shutil.copytree(cache_node_dir, to_dir) initialize_datadir(self.options.tmpdir, i) # Overwrite port/rpcport in bitcoin.conf def _initialize_chain_clean(self): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index b9d1082ddc..3311377090 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -23,6 +23,7 @@ import sys from .authproxy import JSONRPCException from .util import ( + MAX_NODES, append_config, delete_cookie_file, get_rpc_proxy, @@ -110,10 +111,8 @@ class TestNode(): self.p2ps = [] - def get_deterministic_priv_key(self): - """Return a deterministic priv key in base58, that only depends on the node's index""" - AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) - PRIV_KEYS = [ + AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) + PRIV_KEYS = [ # address , privkey AddressKeyPair('mjTkW3DjgyZck4KbiRusZsqTgaYTxdSz6z', 'cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW'), AddressKeyPair('msX6jQXvxiNhx3Q62PKeLPrhrqZQdSimTg', 'cUxsWyKyZ9MAQTaAhUQWJmBbSvHMwSmuv59KgxQV7oZQU3PXN3KE'), @@ -124,8 +123,15 @@ class TestNode(): AddressKeyPair('myzuPxRwsf3vvGzEuzPfK9Nf2RfwauwYe6', 'cQMpDLJwA8DBe9NcQbdoSb1BhmFxVjWD5gRyrLZCtpuF9Zi3a9RK'), AddressKeyPair('mumwTaMtbxEPUswmLBBN3vM9oGRtGBrys8', 'cSXmRKXVcoouhNNVpcNKFfxsTsToY5pvB9DVsFksF1ENunTzRKsy'), AddressKeyPair('mpV7aGShMkJCZgbW7F6iZgrvuPHjZjH9qg', 'cSoXt6tm3pqy43UMabY6eUTmR3eSUYFtB2iNQDGgb3VUnRsQys2k'), - ] - return PRIV_KEYS[self.index] + AddressKeyPair('mq4fBNdckGtvY2mijd9am7DRsbRB4KjUkf', 'cN55daf1HotwBAgAKWVgDcoppmUNDtQSfb7XLutTLeAgVc3u8hik'), + AddressKeyPair('mpFAHDjX7KregM3rVotdXzQmkbwtbQEnZ6', 'cT7qK7g1wkYEMvKowd2ZrX1E5f6JQ7TM246UfqbCiyF7kZhorpX3'), + AddressKeyPair('mzRe8QZMfGi58KyWCse2exxEFry2sfF2Y7', 'cPiRWE8KMjTRxH1MWkPerhfoHFn5iHPWVK5aPqjW8NxmdwenFinJ'), + ] + + def get_deterministic_priv_key(self): + """Return a deterministic priv key in base58, that only depends on the node's index""" + assert len(self.PRIV_KEYS) == MAX_NODES + return self.PRIV_KEYS[self.index] def get_mem_rss_kilobytes(self): """Get the memory usage (RSS) per `ps`. diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 0583b42388..26215083fb 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -228,7 +228,7 @@ def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=N ############################################ # The maximum number of nodes a single test can spawn -MAX_NODES = 8 +MAX_NODES = 12 # Don't assign rpc or p2p ports lower than this PORT_MIN = 11000 # The number of ports to "reserve" for p2p and rpc, each diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 4d1f1ccdc1..15f2195e21 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -28,12 +28,17 @@ def create_transactions(node, address, amt, fees): for utxo in utxos: inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]}) ins_total += utxo['amount'] - if ins_total + max(fees) > amt: + if ins_total >= amt + max(fees): break + # make sure there was enough utxos + assert ins_total >= amt + max(fees) txs = [] for fee in fees: - outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee} + outputs = {address: amt} + # prevent 0 change output + if ins_total > amt + fee: + outputs[node.getrawchangeaddress()] = ins_total - amt - fee raw_tx = node.createrawtransaction(inputs, outputs, 0, True) raw_tx = node.signrawtransactionwithwallet(raw_tx) assert_equal(raw_tx['complete'], True) |