diff options
30 files changed, 366 insertions, 242 deletions
diff --git a/.travis.yml b/.travis.yml index 64227ac2a8..af9c476dc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ env: - CCACHE_TEMPDIR=/tmp/.ccache-temp - CCACHE_COMPRESS=1 - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks + - SDK_URL=https://bitcoin.jonasschnelli.ch/sdks - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all matrix: @@ -39,7 +39,7 @@ env: # No wallet - HOST=x86_64-unknown-linux-gnu PACKAGES=" openjdk-7-jre-headless python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Cross-Mac - - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" + - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" before_install: - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") @@ -8,7 +8,7 @@ https://bitcoincore.org What is Bitcoin? ---------------- -Bitcoin is an experimental new digital currency that enables instant payments to +Bitcoin is an experimental digital currency that enables instant payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate with no central authority: managing transactions and issuing money are carried out collectively by the network. Bitcoin Core is the name of open source diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index efffa4887d..74d9102674 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -342,6 +342,10 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ elif test x$TARGET_OS = xdarwin; then PKG_CHECK_MODULES([QTPRINT], [Qt5PrintSupport], [QT_LIBS="$QTPRINT_LIBS $QT_LIBS"]) fi + else + if ${PKG_CONFIG} --exists "Qt5Core >= 5.6" 2>/dev/null; then + QT_LIBS="-lQt5PlatformSupport $QT_LIBS" + fi fi ]) else diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 536fcfb105..991976d59e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -31,7 +31,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "MacOSX10.9.sdk.tar.gz" +- "MacOSX10.11.sdk.tar.gz" script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin11" @@ -87,7 +87,7 @@ script: | BASEPREFIX=`pwd`/depends mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.9.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md new file mode 100644 index 0000000000..e9e3f65da2 --- /dev/null +++ b/contrib/verify-commits/README.md @@ -0,0 +1,26 @@ +Tooling for verification of PGP signed commits +---------------------------------------------- + +This is an incomplete work in progress, but currently includes a pre-push hook +script (`pre-push-hook.sh`) for maintainers to ensure that their own commits +are PGP signed (nearly always merge commits), as well as a script to verify +commits against a trusted keys list. + + +Using verify-commits.sh safely +------------------------------ + +Remember that you can't use an untrusted script to verify itself. This means +that checking out code, then running `verify-commits.sh` against `HEAD` is +_not_ safe, because the version of `verify-commits.sh` that you just ran could +be backdoored. Instead, you need to use a trusted version of verify-commits +prior to checkout to make sure you're checking out only code signed by trusted +keys: + + git fetch origin && \ + ./contrib/verify-commits/verify-commits.sh origin/master && \ + git checkout origin/master + +Note that the above isn't a good UI/UX yet, and needs significant improvements +to make it more convenient and reduce the chance of errors; pull-reqs +improving this process would be much appreciated. diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index 31aeb8f3d3..e69de29bb2 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -1,2 +0,0 @@ -586a29253dabec3ca0f1ccba9091daabd16b8411 -eddaba7b5692288087a926da5733e86b47274e4e diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 0218b82e11..375d711725 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -1,8 +1,9 @@ #!/bin/sh -INPUT=$(</dev/stdin) +INPUT=$(cat /dev/stdin) VALID=false REVSIG=false -IFS=$'\n' +IFS=' +' for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do case "$LINE" in "[GNUPG:] VALIDSIG "*) @@ -13,10 +14,9 @@ for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do "[GNUPG:] REVKEYSIG "*) [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 while read KEY; do - case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY:24:40} "*) + case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*) REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${KEY:24:40} " - ;; + GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} " esac done < ./contrib/verify-commits/trusted-keys ;; diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index 838b8d1ea8..c60f8ab695 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -165e323d851cc87213c7673c6f278e87a6f2e752 +82bcf405f6db1d55b684a1f63a4aabad376cdad7 diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index ad1b28be0c..75242c2a97 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,8 +1,4 @@ 71A3B16735405025D447E8F274810B012346C9A6 -1F4410F6A89268CE3197A84C57896D2FF8F0B657 -01CDF4627A3B88AAE4A571C87588242FBE38D3A8 -AF8BE07C7049F3A26B239D5325B3083201782B2F -81291FA67D2C379A006A053FEAB5AF94D9E9ABE7 3F1888C6DCA92A6499C4911FDBA1A67379A1A931 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC FE09B823E6D83A3BC7983EAA2D7F2372E50FE137 diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index 9ba781008a..5219331e2e 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -1,25 +1,19 @@ #!/bin/sh +# Not technically POSIX-compliant due to use of "local", but almost every +# shell anyone uses today supports it, so its probably fine DIR=$(dirname "$0") - -echo "Please verify all commits in the following list are not evil:" -git log "$DIR" +[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0") VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") - -IS_REVSIG_ALLOWED () { - while read LINE; do - [ "$LINE" = "$1" ] && return 0 - done < "${DIR}/allow-revsig-commits" - return 1 -} +REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") HAVE_FAILED=false IS_SIGNED () { if [ $1 = $VERIFIED_ROOT ]; then return 0; fi - if IS_REVSIG_ALLOWED "$1"; then + if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 else export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 @@ -27,7 +21,8 @@ IS_SIGNED () { if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then return 1; fi - local PARENTS=$(git show -s --format=format:%P $1) + local PARENTS + PARENTS=$(git show -s --format=format:%P $1) for PARENT in $PARENTS; do if IS_SIGNED $PARENT > /dev/null; then return 0; diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index dbe6d00795..985649619f 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,7 +1,7 @@ OSX_MIN_VERSION=10.7 -OSX_SDK_VERSION=10.9 +OSX_SDK_VERSION=10.11 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -LD64_VERSION=241.9 +LD64_VERSION=253.9 darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index b5603a8d48..797480c25e 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,14 +1,14 @@ package=native_cctools -$(package)_version=ee31ae567931c426136c94aad457c7b51d844beb +$(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6 $(package)_download_path=https://github.com/theuni/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ef107e6ab1b3994cb22e14f4f5c59ea0c0b5a988e6b21d42ed9616b018bbcbf9 +$(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a $(package)_build_subdir=cctools -$(package)_clang_version=3.3 +$(package)_clang_version=3.7.1 $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_version) -$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz -$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz -$(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9 $(package)_extra_sources=$($(package)_clang_file_name) define $(package)_fetch_cmds @@ -23,6 +23,7 @@ define $(package)_extract_cmds $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ + rm -f toolchain/lib/libc++abi.so* && \ echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ chmod +x toolchain/bin/$(host)-dsymutil && \ @@ -30,7 +31,7 @@ define $(package)_extract_cmds endef define $(package)_set_vars -$(package)_config_opts=--target=$(host) --disable-libuuid +$(package)_config_opts=--target=$(host) --disable-lto-support $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ @@ -53,10 +54,11 @@ define $(package)_stage_cmds cd $($(package)_extract_dir)/toolchain && \ mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ - cp -P bin/clang bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ + cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ + cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/$(host)-dsymutil $($(package)_staging_prefix_dir)/bin && \ + cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi endef diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk index c6452820a2..5ee9f17a63 100644 --- a/depends/packages/openssl.mk +++ b/depends/packages/openssl.mk @@ -6,9 +6,42 @@ $(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fce define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" -$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl no-zlib no-shared no-dso -$(package)_config_opts+=no-krb5 no-camellia no-capieng no-cast no-cms no-dtls1 no-gost no-gmp no-heartbeats no-idea no-jpake no-md2 -$(package)_config_opts+=no-mdc2 no-rc5 no-rdrand no-rfc3779 no-rsax no-sctp no-seed no-sha0 no-static_engine no-whirlpool no-rc2 no-rc4 no-ssl2 no-ssl3 +$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl +$(package)_config_opts+=no-camellia +$(package)_config_opts+=no-capieng +$(package)_config_opts+=no-cast +$(package)_config_opts+=no-comp +$(package)_config_opts+=no-dso +$(package)_config_opts+=no-dtls1 +$(package)_config_opts+=no-ec_nistp_64_gcc_128 +$(package)_config_opts+=no-gost +$(package)_config_opts+=no-gmp +$(package)_config_opts+=no-heartbeats +$(package)_config_opts+=no-idea +$(package)_config_opts+=no-jpake +$(package)_config_opts+=no-krb5 +$(package)_config_opts+=no-libunbound +$(package)_config_opts+=no-md2 +$(package)_config_opts+=no-mdc2 +$(package)_config_opts+=no-rc4 +$(package)_config_opts+=no-rc5 +$(package)_config_opts+=no-rdrand +$(package)_config_opts+=no-rfc3779 +$(package)_config_opts+=no-rsax +$(package)_config_opts+=no-sctp +$(package)_config_opts+=no-seed +$(package)_config_opts+=no-sha0 +$(package)_config_opts+=no-shared +$(package)_config_opts+=no-ssl-trace +$(package)_config_opts+=no-ssl2 +$(package)_config_opts+=no-ssl3 +$(package)_config_opts+=no-static_engine +$(package)_config_opts+=no-store +$(package)_config_opts+=no-unit-test +$(package)_config_opts+=no-weak-ssl-ciphers +$(package)_config_opts+=no-whirlpool +$(package)_config_opts+=no-zlib +$(package)_config_opts+=no-zlib-dynamic $(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) $(package)_config_opts_linux=-fPIC -Wa,--noexecstack $(package)_config_opts_x86_64_linux=linux-x86_64 diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index c1fc8e3058..d41d0b9ea5 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,20 +1,21 @@ PACKAGE=qt -$(package)_version=5.5.0 -$(package)_download_path=http://download.qt.io/official_releases/qt/5.5/$($(package)_version)/submodules +$(package)_version=5.6.1 +$(package)_download_path=http://download.qt.io/official_releases/qt/5.6/$($(package)_version)/submodules $(package)_suffix=opensource-src-$($(package)_version).tar.gz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=7e82b1318f88e56a2a9376e069aa608d4fd96b48cb0e1b880ae658b0a1af0561 +$(package)_sha256_hash=0ac67cf8d66d52b995f96c31c4b48117a1afb3db99eaa93e20ccd8f7f55f7fde $(package)_dependencies=openssl -$(package)_linux_dependencies=freetype fontconfig dbus libxcb libX11 xproto libXext +$(package)_linux_dependencies=freetype fontconfig libxcb libX11 xproto libXext $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=mac-qmake.conf fix-xcb-include-order.patch mingw-uuidof.patch pidlist_absolute.patch +$(package)_patches=mac-qmake.conf mingw-uuidof.patch pidlist_absolute.patch fix-xcb-include-order.patch fix_qt_pkgconfig.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=c4bd6db6e426965c6f8824c54e81f68bbd61e2bae1bcadc328c6e81c45902a0d +$(package)_qttranslations_sha256_hash=dcc1534d247babca1840cb6d0a000671801a341ea352d0535474f86adadaf028 + $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=d9e06bd19ecc86afba5e95d45a906d1bc1ad579aa70001e36143c1aaf695bdd6 +$(package)_qttools_sha256_hash=e0f845de28c31230dfa428f0190ccb3b91d1fc02481b1f064698ae4ef8376aa1 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -22,21 +23,34 @@ $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars $(package)_config_opts_release = -release $(package)_config_opts_debug = -debug -$(package)_config_opts += -opensource -confirm-license +$(package)_config_opts += -bindir $(build_prefix)/bin +$(package)_config_opts += -c++11 +$(package)_config_opts += -confirm-license +$(package)_config_opts += -dbus-runtime +$(package)_config_opts += -hostprefix $(build_prefix) +$(package)_config_opts += -no-alsa $(package)_config_opts += -no-audio-backend +$(package)_config_opts += -no-cups +$(package)_config_opts += -no-egl +$(package)_config_opts += -no-eglfs +$(package)_config_opts += -no-feature-style-windowsmobile +$(package)_config_opts += -no-feature-style-windowsce +$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-gif $(package)_config_opts += -no-glib +$(package)_config_opts += -no-gstreamer $(package)_config_opts += -no-icu -$(package)_config_opts += -no-cups $(package)_config_opts += -no-iconv -$(package)_config_opts += -no-gif -$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-kms +$(package)_config_opts += -no-linuxfb +$(package)_config_opts += -no-libudev +$(package)_config_opts += -no-mitshm +$(package)_config_opts += -no-mtdev $(package)_config_opts += -no-nis -$(package)_config_opts += -pch +$(package)_config_opts += -no-pulseaudio +$(package)_config_opts += -no-openvg +$(package)_config_opts += -no-reduce-relocations $(package)_config_opts += -no-qml-debug -$(package)_config_opts += -nomake examples -$(package)_config_opts += -nomake tests -$(package)_config_opts += -no-feature-style-windowsmobile -$(package)_config_opts += -no-feature-style-windowsce $(package)_config_opts += -no-sql-db2 $(package)_config_opts += -no-sql-ibase $(package)_config_opts += -no-sql-oci @@ -46,36 +60,25 @@ $(package)_config_opts += -no-sql-odbc $(package)_config_opts += -no-sql-psql $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 -$(package)_config_opts += -prefix $(host_prefix) -$(package)_config_opts += -hostprefix $(build_prefix) -$(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++11 +$(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -no-xinput2 +$(package)_config_opts += -no-xrender +$(package)_config_opts += -nomake examples +$(package)_config_opts += -nomake tests +$(package)_config_opts += -opensource $(package)_config_opts += -openssl-linked -$(package)_config_opts += -v -$(package)_config_opts += -static -$(package)_config_opts += -silent +$(package)_config_opts += -optimized-qmake +$(package)_config_opts += -pch $(package)_config_opts += -pkg-config +$(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng $(package)_config_opts += -qt-libjpeg -$(package)_config_opts += -qt-zlib $(package)_config_opts += -qt-pcre -$(package)_config_opts += -no-pulseaudio -$(package)_config_opts += -no-openvg -$(package)_config_opts += -no-xrender -$(package)_config_opts += -no-alsa -$(package)_config_opts += -no-mtdev -$(package)_config_opts += -no-gstreamer -$(package)_config_opts += -no-mitshm -$(package)_config_opts += -no-kms -$(package)_config_opts += -no-reduce-relocations -$(package)_config_opts += -no-egl -$(package)_config_opts += -no-eglfs -$(package)_config_opts += -no-linuxfb -$(package)_config_opts += -no-xinput2 -$(package)_config_opts += -no-libudev -$(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -qt-zlib $(package)_config_opts += -reduce-exports -$(package)_config_opts += -optimized-qmake +$(package)_config_opts += -static +$(package)_config_opts += -silent +$(package)_config_opts += -v ifneq ($(build_os),darwin) $(package)_config_opts_darwin = -xplatform macx-clang-linux @@ -119,19 +122,22 @@ define $(package)_extract_cmds tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef + define $(package)_preprocess_cmds sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ sed -i.old "s/src_plugins.depends = src_sql src_xml src_network/src_plugins.depends = src_xml src_network/" qtbase/src/src.pro && \ sed -i.old "s|X11/extensions/XIproto.h|X11/X.h|" qtbase/src/plugins/platforms/xcb/qxcbxsettings.cpp && \ sed -i.old 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' qtbase/configure && \ + sed -i.old 's/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, 0)/CGEventCreateMouseEvent(0, kCGEventMouseMoved, pos, kCGMouseButtonLeft)/' qtbase/src/plugins/platforms/cocoa/qcocoacursor.mm && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ - patch -p1 < $($(package)_patch_dir)/fix-xcb-include-order.patch && \ patch -p1 < $($(package)_patch_dir)/mingw-uuidof.patch && \ patch -p1 < $($(package)_patch_dir)/pidlist_absolute.patch && \ + patch -p1 < $($(package)_patch_dir)/fix-xcb-include-order.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ echo "QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ diff --git a/depends/patches/qt/fix-xcb-include-order.patch b/depends/patches/qt/fix-xcb-include-order.patch index ae469ea94b..c7dbebedce 100644 --- a/depends/patches/qt/fix-xcb-include-order.patch +++ b/depends/patches/qt/fix-xcb-include-order.patch @@ -1,15 +1,15 @@ --- old/qtbase/src/plugins/platforms/xcb/xcb_qpa_lib.pro 2015-03-17 02:06:42.705930685 +0000 +++ new/qtbase/src/plugins/platforms/xcb/xcb_qpa_lib.pro 2015-03-17 02:08:41.281926351 +0000 -@@ -94,8 +94,6 @@ - +@@ -74,8 +74,6 @@ + DEFINES += $$QMAKE_DEFINES_XCB LIBS += $$QMAKE_LIBS_XCB -QMAKE_CXXFLAGS += $$QMAKE_CFLAGS_XCB -QMAKE_CFLAGS += $$QMAKE_CFLAGS_XCB - + CONFIG += qpa/genericunixfontdatabase - -@@ -104,7 +102,8 @@ + +@@ -87,7 +85,8 @@ contains(QT_CONFIG, xcb-qt) { DEFINES += XCB_USE_RENDER XCB_DIR = ../../../3rdparty/xcb @@ -18,28 +18,32 @@ + QMAKE_CXXFLAGS += -I$$XCB_DIR/include -I$$XCB_DIR/sysinclude $$QMAKE_CFLAGS_XCB LIBS += -lxcb -L$$OUT_PWD/xcb-static -lxcb-static } else { - LIBS += -lxcb -lxcb-image -lxcb-icccm -lxcb-sync -lxcb-xfixes -lxcb-shm -lxcb-randr -lxcb-shape -lxcb-keysyms + LIBS += -lxcb -lxcb-image -lxcb-icccm -lxcb-sync -lxcb-xfixes -lxcb-shm -lxcb-randr -lxcb-shape -lxcb-keysyms -lxcb-xinerama --- old/qtbase/src/plugins/platforms/xcb/xcb-static/xcb-static.pro 2015-03-17 02:07:04.641929383 +0000 +++ new/qtbase/src/plugins/platforms/xcb/xcb-static/xcb-static.pro 2015-03-17 02:10:15.485922059 +0000 -@@ -8,7 +8,8 @@ - +@@ -9,7 +9,8 @@ + XCB_DIR = ../../../../3rdparty/xcb - + -INCLUDEPATH += $$XCB_DIR/include $$XCB_DIR/include/xcb $$XCB_DIR/sysinclude +QMAKE_CFLAGS += -I$$XCB_DIR/include -I$$XCB_DIR/include/xcb -I$$XCB_DIR/sysinclude +QMAKE_CXXFLAGS += -I$$XCB_DIR/include -I$$XCB_DIR/include/xcb -I$$XCB_DIR/sysinclude - + QMAKE_CXXFLAGS += $$QMAKE_CFLAGS_XCB QMAKE_CFLAGS += $$QMAKE_CFLAGS_XCB --- old/qtbase/src/plugins/platforms/xcb/xcb-plugin.pro 2015-07-24 16:02:59.530038830 -0400 +++ new/qtbase/src/plugins/platforms/xcb/xcb-plugin.pro 2015-07-24 16:01:22.106037459 -0400 -@@ -11,3 +11,9 @@ +@@ -6,6 +6,13 @@ qxcbmain.cpp OTHER_FILES += xcb.json README - + +contains(QT_CONFIG, xcb-qt) { + DEFINES += XCB_USE_RENDER + XCB_DIR = ../../../3rdparty/xcb + QMAKE_CFLAGS += -I$$XCB_DIR/include -I$$XCB_DIR/sysinclude $$QMAKE_CFLAGS_XCB + QMAKE_CXXFLAGS += -I$$XCB_DIR/include -I$$XCB_DIR/sysinclude $$QMAKE_CFLAGS_XCB +} ++ + PLUGIN_TYPE = platforms + PLUGIN_CLASS_NAME = QXcbIntegrationPlugin + !equals(TARGET, $$QT_DEFAULT_QPA_PLUGIN): PLUGIN_EXTENDS = - diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch new file mode 100644 index 0000000000..3772db4f8b --- /dev/null +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -0,0 +1,11 @@ +--- old/qtbase/mkspecs/features/qt_module.prf 2016-03-17 02:06:42.705930685 +0000 ++++ new/qtbase/mkspecs/features/qt_module.prf 2016-03-17 02:06:42.705930685 +0000 +@@ -244,7 +244,7 @@ + load(qt_targets) + + # this builds on top of qt_common +-!internal_module:!lib_bundle:if(unix|mingw) { ++unix|mingw { + CONFIG += create_pc + QMAKE_PKGCONFIG_DESTDIR = pkgconfig + host_build: \ diff --git a/doc/README_osx.txt b/doc/README_osx.md index c13efaa145..aed3cd97e1 100644 --- a/doc/README_osx.txt +++ b/doc/README_osx.md @@ -1,21 +1,19 @@ Deterministic OS X Dmg Notes. Working OS X DMGs are created in Linux by combining a recent clang, -the Apple's binutils (ld, ar, etc), and DMG authoring tools. +the Apple binutils (ld, ar, etc) and DMG authoring tools. Apple uses clang extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of -F, -target, -mmacosx-version-min, and --sysroot, which are all necessary -when building for OS X. A pre-compiled version of 3.2 is used because it was not -available in the Precise repositories at the time this work was started. In the -future, it can be switched to use system packages instead. +when building for OS X. Apple's version of binutils (called cctools) contains lots of functionality missing in the FSF's binutils. In addition to extra linker options for frameworks and sysroots, several other tools are needed as well such as install_name_tool, lipo, and nmedit. These do not build under linux, so they have been patched to do so. The work here was used as a starting point: -https://github.com/mingwandroid/toolchain4 +[mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). In order to build a working toolchain, the following source packages are needed from Apple: cctools, dyld, and ld64. @@ -29,16 +27,19 @@ originally done in toolchain4. To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. -To obtain it, register for a developer account, then download the Xcode 6.1.1 dmg: -https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg +To obtain it, register for a developer account, then download the [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). This file is several gigabytes in size, but only a single directory inside is -needed: Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.9.sdk +needed: +``` +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk +``` Unfortunately, the usual linux tools (7zip, hpmount, loopback mount) are incapable of opening this file. To create a tarball suitable for Gitian input, mount the dmg in OS X, then create it with: - $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk - +``` + $ tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk +``` The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are created using these tools. The build process has been designed to @@ -48,15 +49,14 @@ fully deterministic and may be freely redistributed. genisoimage is used to create the initial DMG. It is not deterministic as-is, so it has been patched. A system genisoimage will work fine, but it will not be deterministic because the file-order will change between invocations. -The patch can be seen here: -https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff +The patch can be seen here: [theuni/osx-cross-depends](https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff). No effort was made to fix this cleanly, so it likely leaks memory badly. But it's only used for a single invocation, so that's no real concern. genisoimage cannot compress DMGs, so afterwards, the 'dmg' tool from the libdmg-hfsplus project is used to compress it. There are several bugs in this tool and its maintainer has seemingly abandoned the project. It has been forked -and is available (with fixes) here: https://github.com/theuni/libdmg-hfsplus . +and is available (with fixes) here: [theuni/libdmg-hfsplus](https://github.com/theuni/libdmg-hfsplus). The 'dmg' tool has the ability to create DMGs from scratch as well, but this functionality is broken. Only the compression feature is currently used. @@ -77,6 +77,6 @@ build process to remain somewhat deterministic. Here's how it works: that have been previously (deterministically) built in order to create a final dmg. - The Apple keyholder uses this unsigned app to create a detached signature, - using the script that is also included there. + using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). - Builders feed the unsigned app + detached signature back into Gitian. It uses the pre-built tools to recombine the pieces into a deterministic dmg. diff --git a/doc/build-osx.md b/doc/build-osx.md index 89d7816c9a..c9eb4225ab 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -5,11 +5,13 @@ The built-in one is located in `/Applications/Utilities/Terminal.app`. Preparation ----------- -Download and install [Xcode](https://developer.apple.com/xcode/download). +Install the OS X command line tools: -Once installed, run `xcode-select --install` to install the OS X command line tools. +`xcode-select --install` -Install [Homebrew](http://brew.sh). +When the popup appears, click `Install`. + +Then install [Homebrew](http://brew.sh). Dependencies ---------------------- diff --git a/doc/release-process.md b/doc/release-process.md index 3bfcc38177..f5d0d836d1 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -3,7 +3,7 @@ Release Process Before every release candidate: -* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#syncing-with-transifex) +* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). Before every minor and major release: @@ -13,7 +13,7 @@ Before every minor and major release: Before every major release: -* Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. +* Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. ### First time / New builders @@ -75,7 +75,7 @@ Ensure your gitian.sigs are up-to-date if you wish to gverify your builds agains git pull popd -Ensure gitian-builder is up-to-date to take advantage of new caching features (`e9741525c` or later is recommended). +Ensure gitian-builder is up-to-date: pushd ./gitian-builder git pull @@ -89,13 +89,7 @@ Ensure gitian-builder is up-to-date to take advantage of new caching features (` wget -P inputs http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz popd -Register and download the Apple SDK: see [OS X readme](README_osx.txt) for details. - -https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg - -Using a Mac, create a tarball for the 10.9 SDK and copy it to the inputs directory: - - tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk +Create the OS X SDK tarball, see the [OS X readme](README_osx.md) for details, and copy it into the inputs directory. ### Optional: Seed the Gitian sources cache and offline git repositories @@ -239,7 +233,7 @@ transmission-show -m <torrent file> Insert the magnet URI into the announcement sent to mailing lists. This permits people without access to `bitcoin.org` to download the binary distribution. Also put it into the `optional_magnetlink:` slot in the YAML file for -bitcoin.org (see below for bitcoin.org update instructions). +bitcoin.org (see below for bitcoin.org update instructions). - Update bitcoin.org version diff --git a/doc/tor.md b/doc/tor.md index 43e922718b..79f1563021 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -95,12 +95,22 @@ Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket API, to create and destroy 'ephemeral' hidden services programmatically. Bitcoin Core has been updated to make use of this. -This means that if Tor is running (and proper authorization is available), -Bitcoin Core automatically creates a hidden service to listen on, without -manual configuration. This will positively affect the number of available -.onion nodes. +This means that if Tor is running (and proper authentication has been configured), +Bitcoin Core automatically creates a hidden service to listen on. This will positively +affect the number of available .onion nodes. This new feature is enabled by default if Bitcoin Core is listening, and a connection to Tor can be made. It can be configured with the `-listenonion`, `-torcontrol` and `-torpassword` settings. To show verbose debugging information, pass `-debug=tor`. + +Connecting to Tor's control socket API requires one of two authentication methods to be +configured. For cookie authentication the user running bitcoind must have write access +to the `CookieAuthFile` specified in Tor configuration. In some cases this is +preconfigured and the creation of a hidden service is automatic. If permission problems +are seen with `-debug=tor` they can be resolved by adding both the user running tor and +the user running bitcoind to the same group and setting permissions appropriately. On +Debian-based systems the user running bitcoind can be added to the debian-tor group, +which has the appropriate permissions. An alternative authentication method is the use +of the `-torpassword` flag and a `hash-password` which can be enabled and specified in +Tor configuration.
\ No newline at end of file diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 57a576f1c7..6c0ed4510f 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -101,6 +101,8 @@ if ENABLE_ZMQ: #Tests testScripts = [ + # longest test should go first, to favor running tests in parallel + 'p2p-fullblocktest.py', 'walletbackup.py', 'bip68-112-113-p2p.py', 'wallet.py', @@ -125,7 +127,6 @@ testScripts = [ 'nodehandling.py', 'reindex.py', 'decodescript.py', - 'p2p-fullblocktest.py', 'blockchain.py', 'disablewallet.py', 'sendheaders.py', @@ -191,7 +192,7 @@ def runtests(): if coverage: flags.append(coverage.flag) - if len(test_list) > 1: + if len(test_list) > 1 and run_parallel > 1: # Populate cache subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) @@ -251,7 +252,7 @@ class RPCTestHandler: stdout=subprocess.PIPE, stderr=subprocess.PIPE))) if not self.jobs: - raise IndexError('%s from empty list' % __name__) + raise IndexError('pop from empty list') while True: # Return first proc that finishes time.sleep(.5) diff --git a/qa/rpc-tests/fundrawtransaction.py b/qa/rpc-tests/fundrawtransaction.py index 228574e671..eeb8476634 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/qa/rpc-tests/fundrawtransaction.py @@ -6,7 +6,14 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -# Create one-input, one-output, no-fee transaction: + +def get_unspent(listunspent, amount): + for utx in listunspent: + if utx['amount'] == amount: + return utx + raise AssertionError('Could not find unspent with amount={}'.format(amount)) + + class RawTransactionsTest(BitcoinTestFramework): def __init__(self): @@ -71,7 +78,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test if we have enought inputs + assert(len(dec_tx['vin']) > 0) #test that we have enough inputs ############################## # simple test with two coins # @@ -123,14 +130,7 @@ class RawTransactionsTest(BitcoinTestFramework): ######################################################################### # test a fundrawtransaction with a VIN greater than the required amount # ######################################################################### - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 5.0: - utx = aUtx - break - - assert(utx!=False) + utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] outputs = { self.nodes[0].getnewaddress() : 1.0 } @@ -151,14 +151,7 @@ class RawTransactionsTest(BitcoinTestFramework): ##################################################################### # test a fundrawtransaction with which will not get a change output # ##################################################################### - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 5.0: - utx = aUtx - break - - assert(utx!=False) + utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] outputs = { self.nodes[0].getnewaddress() : Decimal(5.0) - fee - feeTolerance } @@ -180,14 +173,7 @@ class RawTransactionsTest(BitcoinTestFramework): #################################################### # test a fundrawtransaction with an invalid option # #################################################### - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 5.0: - utx = aUtx - break - - assert_equal(utx!=False, True) + utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } @@ -205,14 +191,7 @@ class RawTransactionsTest(BitcoinTestFramework): ############################################################ # test a fundrawtransaction with an invalid change address # ############################################################ - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 5.0: - utx = aUtx - break - - assert_equal(utx!=False, True) + utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } @@ -227,18 +206,10 @@ class RawTransactionsTest(BitcoinTestFramework): assert("changeAddress must be a valid bitcoin address" in e.error['message']) - ############################################################ # test a fundrawtransaction with a provided change address # ############################################################ - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 5.0: - utx = aUtx - break - - assert_equal(utx!=False, True) + utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } @@ -247,24 +218,22 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() + try: + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 2}) + except JSONRPCException as e: + assert('changePosition out of bounds' == e.error['message']) + else: + assert(False) rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0]; assert_equal(change, out['scriptPubKey']['addresses'][0]) - ######################################################################### # test a fundrawtransaction with a VIN smaller than the required amount # ######################################################################### - utx = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 1.0: - utx = aUtx - break - - assert(utx!=False) + utx = get_unspent(self.nodes[2].listunspent(), 1) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] outputs = { self.nodes[0].getnewaddress() : 1.0 } @@ -299,17 +268,8 @@ class RawTransactionsTest(BitcoinTestFramework): ########################################### # test a fundrawtransaction with two VINs # ########################################### - utx = False - utx2 = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 1.0: - utx = aUtx - if aUtx['amount'] == 5.0: - utx2 = aUtx - - - assert(utx!=False) + utx = get_unspent(self.nodes[2].listunspent(), 1) + utx2 = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']},{'txid' : utx2['txid'], 'vout' : utx2['vout']} ] outputs = { self.nodes[0].getnewaddress() : 6.0 } @@ -341,17 +301,8 @@ class RawTransactionsTest(BitcoinTestFramework): ######################################################### # test a fundrawtransaction with two VINs and two vOUTs # ######################################################### - utx = False - utx2 = False - listunspent = self.nodes[2].listunspent() - for aUtx in listunspent: - if aUtx['amount'] == 1.0: - utx = aUtx - if aUtx['amount'] == 5.0: - utx2 = aUtx - - - assert(utx!=False) + utx = get_unspent(self.nodes[2].listunspent(), 1) + utx2 = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']},{'txid' : utx2['txid'], 'vout' : utx2['vout']} ] outputs = { self.nodes[0].getnewaddress() : 6.0, self.nodes[0].getnewaddress() : 1.0 } diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 782df52d62..32fe79efc3 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -156,17 +156,22 @@ def initialize_datadir(dirname, n): datadir = os.path.join(dirname, "node"+str(n)) if not os.path.isdir(datadir): os.makedirs(datadir) + rpc_u, rpc_p = rpc_auth_pair(n) with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f: f.write("regtest=1\n") - f.write("rpcuser=rt\n") - f.write("rpcpassword=rt\n") + f.write("rpcuser=" + rpc_u + "\n") + f.write("rpcpassword=" + rpc_p + "\n") f.write("port="+str(p2p_port(n))+"\n") f.write("rpcport="+str(rpc_port(n))+"\n") f.write("listenonion=0\n") return datadir +def rpc_auth_pair(n): + return 'rpcuser💻' + str(n), 'rpcpass🔑' + str(n) + def rpc_url(i, rpchost=None): - return "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i)) + rpc_u, rpc_p = rpc_auth_pair(i) + return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, rpchost or '127.0.0.1', rpc_port(i)) def wait_for_bitcoind_start(process, url, i): ''' diff --git a/src/main.cpp b/src/main.cpp index bdb3457f8e..361526f337 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -88,12 +88,22 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CTxMemPool mempool(::minRelayTxFee); FeeFilterRounder filterRounder(::minRelayTxFee); +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) + { + return &(*a) < &(*b); + } +}; + struct COrphanTx { CTransaction tx; NodeId fromPeer; + int64_t nTimeExpire; }; map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); -map<uint256, set<uint256> > mapOrphanTransactionsByPrev GUARDED_BY(cs_main); +map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** @@ -623,40 +633,42 @@ bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(c // large transaction with a missing parent then we assume // it will rebroadcast it later, after the parent transaction(s) // have been mined or received. - // 10,000 orphans, each of which is at most 5,000 bytes big is - // at most 500 megabytes of orphans: + // 100 orphans, each of which is at most 99,999 bytes big is + // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); - if (sz > 5000) + if (sz >= MAX_STANDARD_TX_SIZE) { LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } - mapOrphanTransactions[hash].tx = tx; - mapOrphanTransactions[hash].fromPeer = peer; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + assert(ret.second); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); + } - LogPrint("mempool", "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(), + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } -void static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); if (it == mapOrphanTransactions.end()) - return; + return 0; BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) { - map<uint256, set<uint256> >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash); + auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itPrev == mapOrphanTransactionsByPrev.end()) continue; - itPrev->second.erase(hash); + itPrev->second.erase(it); if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } mapOrphanTransactions.erase(it); + return 1; } void EraseOrphansFor(NodeId peer) @@ -668,8 +680,7 @@ void EraseOrphansFor(NodeId peer) map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - EraseOrphanTx(maybeErase->second.tx.GetHash()); - ++nErased; + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); } } if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); @@ -679,6 +690,26 @@ void EraseOrphansFor(NodeId peer) unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { unsigned int nEvicted = 0; + static int64_t nNextSweep; + int64_t nNow = GetTime(); + if (nNextSweep <= nNow) { + // Sweep out expired orphan pool entries: + int nErased = 0; + int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; + if (maybeErase->second.nTimeExpire <= nNow) { + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); + } else { + nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); + } + } + // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. + nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + } while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: @@ -2335,6 +2366,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); + std::vector<uint256> vOrphanErase; std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; @@ -2367,6 +2399,17 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; } + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } + } + if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); @@ -2454,6 +2497,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); hashPrevBestCoinBase = block.vtx[0].GetHash(); + // Erase orphan transactions include or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + } + int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -5041,7 +5093,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; } - vector<uint256> vWorkQueue; + deque<COutPoint> vWorkQueue; vector<uint256> vEraseQueue; CTransaction tx; vRecv >> tx; @@ -5060,7 +5112,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { mempool.check(pcoinsTip); RelayTransaction(tx); - vWorkQueue.push_back(inv.hash); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + vWorkQueue.emplace_back(inv.hash, i); + } pfrom->nLastTXTime = GetTime(); @@ -5071,18 +5125,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; - for (unsigned int i = 0; i < vWorkQueue.size(); i++) - { - map<uint256, set<uint256> >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]); + while (!vWorkQueue.empty()) { + auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); + vWorkQueue.pop_front(); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (set<uint256>::iterator mi = itByPrev->second.begin(); + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const uint256& orphanHash = *mi; - const CTransaction& orphanTx = mapOrphanTransactions[orphanHash].tx; - NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer; + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + NodeId fromPeer = (*mi)->second.fromPeer; bool fMissingInputs2 = false; // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get @@ -5095,7 +5149,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanTx); - vWorkQueue.push_back(orphanHash); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + vWorkQueue.emplace_back(orphanHash, i); + } vEraseQueue.push_back(orphanHash); } else if (!fMissingInputs2) @@ -5124,13 +5180,29 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } else if (fMissingInputs) { - AddOrphanTx(tx, pfrom->GetId()); + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + if (recentRejects->contains(txin.prevout.hash)) { + fRejectedParents = true; + break; + } + } + if (!fRejectedParents) { + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + CInv inv(MSG_TX, txin.prevout.hash); + pfrom->AddInventoryKnown(inv); + if (!AlreadyHave(inv)) pfrom->AskFor(inv); + } + AddOrphanTx(tx, pfrom->GetId()); - // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + // DoS prevention: do not allow mapOrphanTransactions to grow unbounded + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); + if (nEvicted > 0) + LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + } else { + LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + } } else { assert(recentRejects); recentRejects->insert(tx.GetHash()); diff --git a/src/main.h b/src/main.h index e2bfdfdf6e..a39ffbf56d 100644 --- a/src/main.h +++ b/src/main.h @@ -56,6 +56,10 @@ static const CAmount HIGH_TX_FEE_PER_KB = 0.01 * COIN; static const CAmount HIGH_MAX_TX_FEE = 100 * HIGH_TX_FEE_PER_KB; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; +/** Expiration time for orphan transactions in seconds */ +static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; +/** Minimum time between orphan transactions expire time checks in seconds */ +static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** Default for -limitancestorcount, max number of in-mempool ancestors */ static const unsigned int DEFAULT_ANCESTOR_LIMIT = 25; /** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */ diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 6218ab6ab0..64b5c83d72 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -533,6 +533,9 @@ int main(int argc, char *argv[]) // Generate high-dpi pixmaps QApplication::setAttribute(Qt::AA_UseHighDpiPixmaps); #endif +#if QT_VERSION >= 0x050600 + QGuiApplication::setAttribute(Qt::AA_EnableHighDpiScaling); +#endif #ifdef Q_OS_MAC QApplication::setAttribute(Qt::AA_DontShowIconsInMenus); #endif diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 1bc4f77959..a11da50588 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -27,7 +27,7 @@ bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pM // Warn only once as this is performance-critical static bool warned = false; if (!warned) { - LogPrint("%s: OpenSSL RAND_event() failed to seed OpenSSL PRNG with enough data.\n", __func__); + LogPrintf("%s: OpenSSL RAND_event() failed to seed OpenSSL PRNG with enough data.\n", __func__); warned = true; } } diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 4a373fc60b..93f7ae09da 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) tx.vout.resize(1); tx.vout[0].nValue = 1*CENT; tx.vout[0].scriptPubKey = GetScriptForDestination(key.GetPubKey().GetID()); - tx.vin.resize(500); + tx.vin.resize(2777); for (unsigned int j = 0; j < tx.vin.size(); j++) { tx.vin[j].prevout.n = j; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 205ffd6379..18c54b08bc 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1075,7 +1075,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(0); - while (DynamicMemoryUsage() > sizelimit) { + while (!mapTx.empty() && DynamicMemoryUsage() > sizelimit) { indexed_transaction_set::index<descendant_score>::type::iterator it = mapTx.get<descendant_score>().begin(); // We set the new mempool min fee to the feerate of the removed set, plus the diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 14c2e31d95..d55cc68dc0 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -167,6 +167,11 @@ void ImportScript(const CScript& script, const string& strLabel, bool isRedeemSc if (!pwalletMain->HaveCScript(script) && !pwalletMain->AddCScript(script)) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding p2sh redeemScript to wallet"); ImportAddress(CBitcoinAddress(CScriptID(script)), strLabel); + } else { + CTxDestination destination; + if (ExtractDestination(script, destination)) { + pwalletMain->SetAddressBook(destination, strLabel, "receive"); + } } } @@ -195,6 +200,8 @@ UniValue importaddress(const UniValue& params, bool fHelp) "4. p2sh (boolean, optional, default=false) Add the P2SH version of the script as well\n" "\nNote: This call can take minutes to complete if rescan is true.\n" "If you have the full public key, you should call importpubkey instead of this.\n" + "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" + "as change, and not show up in many RPCs.\n" "\nExamples:\n" "\nImport a script with rescan\n" + HelpExampleCli("importaddress", "\"myscript\"") + |