diff options
40 files changed, 910 insertions, 316 deletions
diff --git a/Makefile.am b/Makefile.am index 6bc0044313..b51f477b78 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,7 +26,9 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/background.png \ - $(top_srcdir)/contrib/macdeploy/DS_Store + $(top_srcdir)/contrib/macdeploy/DS_Store \ + $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ + $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh COVERAGE_INFO = baseline_filtered_combined.info baseline.info block_test.info \ leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \ @@ -85,14 +87,30 @@ if BUILD_DARWIN $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2 +deploydir: $(OSX_DMG) else -$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -verbose 2 - $(MKDIR_P) dist/.background - $(INSTALL) contrib/macdeploy/background.png dist/.background - $(INSTALL) contrib/macdeploy/DS_Store dist/.DS_Store - cd dist; $(LN_S) /Applications Applications - $(GENISOIMAGE) -no-cache-inodes -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o $@ dist +APP_DIST_DIR=$(top_builddir)/dist +APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.png $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications + +$(APP_DIST_DIR)/Applications: + @rm -f $@ + @cd $(@D); $(LN_S) /Applications $(@F) + +$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt + +$(OSX_DMG): $(APP_DIST_EXTRAS) + $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o $@ dist + +$(APP_DIST_DIR)/.background/background.png: + $(MKDIR_P) $(@D) + $(INSTALL) $(top_srcdir)/contrib/macdeploy/background.png $@ +$(APP_DIST_DIR)/.DS_Store: + $(INSTALL) $(top_srcdir)/contrib/macdeploy/DS_Store $@ + +$(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) + INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -translations-dir=$(QT_TRANSLATION_DIR) -add-qt-tr $(OSX_QT_TRANSLATIONS) -verbose 2 + +deploydir: $(APP_DIST_EXTRAS) endif if TARGET_DARWIN diff --git a/configure.ac b/configure.ac index 6784521d81..9814197af3 100644 --- a/configure.ac +++ b/configure.ac @@ -281,6 +281,12 @@ case $host in AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + + dnl libtool will try to strip the static lib, which is a problem for + dnl cross-builds because strip attempts to call a hard-coded ld, + dnl which may not exist in the path. Stripping the .a is not + dnl necessary, so just disable it. + old_striplib= ;; esac fi @@ -421,6 +427,8 @@ AC_CHECK_HEADERS([endian.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/s AC_SEARCH_LIBS([getaddrinfo_a], [anl], [AC_DEFINE(HAVE_GETADDRINFO_A, 1, [Define this symbol if you have getaddrinfo_a])]) AC_SEARCH_LIBS([inet_pton], [nsl resolv], [AC_DEFINE(HAVE_INET_PTON, 1, [Define this symbol if you have inet_pton])]) +AC_CHECK_DECLS([strnlen]) + AC_CHECK_DECLS([le32toh, le64toh, htole32, htole64, be32toh, be64toh, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include <endian.h> diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index b57a04469c..bba2104edb 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -73,13 +73,13 @@ script: | ./autogen.sh ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` make dist - DISTNAME=`echo bitcoin-*.tar.gz` - + SOURCEDIST=`echo bitcoin-*.tar.gz` + DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp - tar xf ../$DISTNAME - find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$DISTNAME + tar xf ../$SOURCEDIST + find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" @@ -88,17 +88,22 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - tar --strip-components=1 -xf ../$DISTNAME + INSTALLPATH=`pwd`/installed/${DISTNAME} + mkdir -p ${INSTALLPATH} + tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${OUTDIR}/${i}/bin --includedir=${OUTDIR}/${i}/include --libdir=${OUTDIR}/${i}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make install-strip - cd .. + cd installed + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + rm -rf ${DISTNAME}/lib/pkgconfig + find . | sort | tar --no-recursion -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + cd ../../ done mkdir -p $OUTDIR/src - mv $DISTNAME $OUTDIR/src - mv ${OUTDIR}/x86_64-* ${OUTDIR}/64 - mv ${OUTDIR}/i686-* ${OUTDIR}/32 + mv $SOURCEDIST $OUTDIR/src + mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-linux64.tar.gz + mv ${OUTDIR}/${DISTNAME}-i686-*.tar.gz ${OUTDIR}/${DISTNAME}-linux32.tar.gz - # Delete unwanted stuff - find ${OUTDIR} -name "lib*.la" -delete diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml new file mode 100644 index 0000000000..db9b4af93d --- /dev/null +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -0,0 +1,37 @@ +--- +name: "bitcoin-dmg-signer" +suites: +- "precise" +architectures: +- "amd64" +packages: +- "libc6:i386" +- "faketime" +reference_datetime: "2013-06-01 00:00:00" +remotes: [] +files: +- "bitcoin-0.9.99-osx-unsigned.tar.gz" +- "signature.tar.gz" +script: | + WRAP_DIR=$HOME/wrapped + mkdir -p ${WRAP_DIR} + export PATH=`pwd`:$PATH + FAKETIME_PROGS="dmg genisoimage" + + # Create global faketime wrappers + for prog in ${FAKETIME_PROGS}; do + echo '#!/bin/bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + + UNSIGNED=`echo bitcoin-*.tar.gz` + SIGNED=`echo ${UNSIGNED} | sed 's/.tar.*//' | sed 's/-unsigned//'`.dmg + + tar -xf ${UNSIGNED} + ./detached-sig-apply.sh ${UNSIGNED} signature.tar.gz + ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "Bitcoin-Qt" -no-pad -r -apple -o uncompressed.dmg signed-app + ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 7ec0700ed3..eb6df2096e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -83,13 +83,14 @@ script: | ./autogen.sh ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` make dist - DISTNAME=`echo bitcoin-*.tar.gz` + SOURCEDIST=`echo bitcoin-*.tar.gz` + DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp - tar xf ../$DISTNAME - find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$DISTNAME + tar xf ../$SOURCEDIST + find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" @@ -98,17 +99,36 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - tar --strip-components=1 -xf ../$DISTNAME + INSTALLPATH=`pwd`/installed/${DISTNAME} + mkdir -p ${INSTALLPATH} + tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${OUTDIR}/${i}/bin --includedir=${OUTDIR}/${i}/include --libdir=${OUTDIR}/${i}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make install-strip + + make deploydir + mkdir -p unsigned-app-${i} + cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} + cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} + cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} + cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate + cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff + mv dist unsigned-app-${i} + pushd unsigned-app-${i} + find . | sort | tar --no-recursion -czf ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz -T - + popd + make deploy - ${WRAP_DIR}/dmg dmg Bitcoin-Qt.dmg ${OUTDIR}/Bitcoin-Qt.dmg - cd .. + ${WRAP_DIR}/dmg dmg Bitcoin-Qt.dmg ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg + + cd installed + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + rm -rf ${DISTNAME}/lib/pkgconfig + find . | sort | tar --no-recursion -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + cd ../../ done mkdir -p $OUTDIR/src - mv $DISTNAME $OUTDIR/src - - # Delete unwanted stuff - find ${OUTDIR} -name "lib*.la" -delete + mv $SOURCEDIST $OUTDIR/src + mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 172807467a..97c823cde6 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -17,6 +17,7 @@ packages: - "mingw-w64" - "g++-mingw-w64" - "nsis" +- "zip" reference_datetime: "2013-06-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" @@ -27,7 +28,7 @@ script: | HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" CONFIGFLAGS="--enable-upnp-default" FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip" - FAKETIME_PROGS="date makensis" + FAKETIME_PROGS="date makensis zip" export QT_RCC_TEST=1 export GZIP="-9n" @@ -75,13 +76,14 @@ script: | ./autogen.sh ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` make dist - DISTNAME=`echo bitcoin-*.tar.gz` + SOURCEDIST=`echo bitcoin-*.tar.gz` + DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp - tar xf ../$DISTNAME - find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$DISTNAME + tar xf ../$SOURCEDIST + find bitcoin-* | sort | tar --no-recursion -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" @@ -90,19 +92,24 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - tar --strip-components=1 -xf ../$DISTNAME + INSTALLPATH=`pwd`/installed/${DISTNAME} + mkdir -p ${INSTALLPATH} + tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${OUTDIR}/${i}/bin --includedir=${OUTDIR}/${i}/include --libdir=${OUTDIR}/${i}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make deploy make install-strip cp -f bitcoin-*setup*.exe $OUTDIR/ - cd .. + cd installed + mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + rm -rf ${DISTNAME}/lib/pkgconfig + find . -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip + cd ../.. done mkdir -p $OUTDIR/src - mv $DISTNAME $OUTDIR/src - mv ${OUTDIR}/x86_64-* ${OUTDIR}/64 - mv ${OUTDIR}/i686-* ${OUTDIR}/32 - - # Delete unwanted stuff - find ${OUTDIR} -name "lib*.la" -delete + mv $SOURCEDIST $OUTDIR/src + mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip + mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip diff --git a/contrib/macdeploy/DS_Store b/contrib/macdeploy/DS_Store Binary files differindex b9a1e14864..7527dc671f 100644 --- a/contrib/macdeploy/DS_Store +++ b/contrib/macdeploy/DS_Store diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh new file mode 100755 index 0000000000..7b3eb1b197 --- /dev/null +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -e + +UNSIGNED=$1 +SIGNATURE=$2 +ARCH=x86_64 +ROOTDIR=dist +BUNDLE=${ROOTDIR}/Bitcoin-Qt.app +TEMPDIR=signed.temp +OUTDIR=signed-app + +if [ -z "$UNSIGNED" ]; then + echo "usage: $0 <unsigned app> <signature>" + exit 1 +fi + +if [ -z "$SIGNATURE" ]; then + echo "usage: $0 <unsigned app> <signature>" + exit 1 +fi + +rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR} +tar -C ${TEMPDIR} -xf ${UNSIGNED} +tar -C ${TEMPDIR} -xf ${SIGNATURE} + +if [ -z "${PAGESTUFF}" ]; then + PAGESTUFF=${TEMPDIR}/pagestuff +fi + +if [ -z "${CODESIGN_ALLOCATE}" ]; then + CODESIGN_ALLOCATE=${TEMPDIR}/codesign_allocate +fi + +for i in `find ${TEMPDIR} -name "*.sign"`; do + SIZE=`stat -c %s ${i}` + TARGET_FILE=`echo ${i} | sed 's/\.sign$//'` + + echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}" + ${CODESIGN_ALLOCATE} -i ${TARGET_FILE} -a ${ARCH} ${SIZE} -o ${i}.tmp + + OFFSET=`${PAGESTUFF} ${i}.tmp -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` + if [ -z ${QUIET} ]; then + echo "Attaching signature at offset ${OFFSET}" + fi + + dd if=$i of=${i}.tmp bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null + mv ${i}.tmp ${TARGET_FILE} + rm ${i} + echo "Success." +done +mv ${TEMPDIR}/${ROOTDIR} ${OUTDIR} +rm -rf ${TEMPDIR} +echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh new file mode 100755 index 0000000000..aff4f08da8 --- /dev/null +++ b/contrib/macdeploy/detached-sig-create.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -e + +ROOTDIR=dist +BUNDLE=${ROOTDIR}/Bitcoin-Qt.app +CODESIGN=codesign +TEMPDIR=sign.temp +TEMPLIST=${TEMPDIR}/signatures.txt +OUT=signature.tar.gz + +if [ ! -n "$1" ]; then + echo "usage: $0 <codesign args>" + echo "example: $0 -s MyIdentity" + exit 1 +fi + +rm -rf ${TEMPDIR} ${TEMPLIST} +mkdir -p ${TEMPDIR} + +${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}" + +for i in `grep -v CodeResources ${TEMPLIST}`; do + TARGETFILE="${BUNDLE}/`echo ${i} | sed "s|.*${BUNDLE}/||"`" + SIZE=`pagestuff $i -p | tail -2 | grep size | sed 's/[^0-9]*//g'` + OFFSET=`pagestuff $i -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` + SIGNFILE="${TEMPDIR}/${TARGETFILE}.sign" + DIRNAME="`dirname ${SIGNFILE}`" + mkdir -p "${DIRNAME}" + echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}" + dd if=$i of=${SIGNFILE} bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null +done + +for i in `grep CodeResources ${TEMPLIST}`; do + TARGETFILE="${BUNDLE}/`echo ${i} | sed "s|.*${BUNDLE}/||"`" + RESOURCE="${TEMPDIR}/${TARGETFILE}" + DIRNAME="`dirname "${RESOURCE}"`" + mkdir -p "${DIRNAME}" + echo "Adding resource for: "${TARGETFILE}"" + cp "${i}" "${RESOURCE}" +done + +rm ${TEMPLIST} + +tar -C ${TEMPDIR} -czf ${OUT} . +rm -rf ${TEMPDIR} +echo "Created ${OUT}" diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 51f2ea663b..6a8e714a48 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -48,6 +48,7 @@ define $(package)_preprocess_cmds sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ sed -i.old "s/src_plugins.depends = src_sql src_xml src_network/src_plugins.depends = src_xml src_network/" qtbase/src/src.pro && \ sed -i.old "/XIproto.h/d" qtbase/src/plugins/platforms/xcb/qxcbxsettings.cpp && \ + sed -i.old 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' qtbase/configure && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/doc/README_osx.txt b/doc/README_osx.txt index 8831649bd8..d56234f7d9 100644 --- a/doc/README_osx.txt +++ b/doc/README_osx.txt @@ -65,3 +65,18 @@ Background images and other features can be added to DMG files by inserting a .DS_Store before creation. The easiest way to create this file is to build a DMG without one, move it to a device running OSX, customize the layout, then grab the .DS_Store file for later use. That is the approach taken here. + +As of OSX Mavericks (10.9), using an Apple-blessed key to sign binaries is a +requirement in order to satisfy the new Gatekeeper requirements. Because this +private key cannot be shared, we'll have to be a bit creative in order for the +build process to remain somewhat deterministic. Here's how it works: + +- Builders use gitian to create an unsigned release. This outputs an unsigned + dmg which users may choose to bless and run. It also outputs an unsigned app + structure in the form of a tarball, which also contains all of the tools + that have been previously (deterministically) built in order to create a + final dmg. +- The Apple keyholder uses this unsigned app to create a detached signature, + using the script that is also included there. +- Builders feed the unsigned app + detached signature back into gitian. It + uses the pre-built tools to recombine the pieces into a deterministic dmg. diff --git a/doc/coding.md b/doc/coding.md index b9d2006e3c..43294dbe4c 100644 --- a/doc/coding.md +++ b/doc/coding.md @@ -115,8 +115,6 @@ Threads - StartNode : Starts other threads. -- ThreadGetMyExternalIP : Determines outside-the-firewall IP address, sends addr message to connected peers when it determines it. - - ThreadDNSAddressSeed : Loads addresses of peers from the DNS. - ThreadMapPort : Universal plug-and-play startup/shutdown diff --git a/doc/release-process.md b/doc/release-process.md index c6a7266ef1..a16d4ace4a 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -54,48 +54,23 @@ Release Process ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - pushd build/out - zip -r bitcoin-${VERSION}-linux-gitian.zip * - mv bitcoin-${VERSION}-linux-gitian.zip ../../../ - popd + mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../ ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-win --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - pushd build/out - zip -r bitcoin-${VERSION}-win-gitian.zip * - mv bitcoin-${VERSION}-win-gitian.zip ../../../ - popd - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-osx --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - pushd build/out - mv Bitcoin-Qt.dmg ../../../ - popd + mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../ + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + mv build/out/bitcoin-*-unsigned.tar.gz inputs + mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../ popd - +bitcoin-0.9.99-osx-unsigned.tar.gz Build output expected: - 1. linux 32-bit and 64-bit binaries + source (bitcoin-${VERSION}-linux-gitian.zip) - 2. windows 32-bit and 64-bit binaries + installer + source (bitcoin-${VERSION}-win-gitian.zip) - 3. OSX installer (Bitcoin-Qt.dmg) - 4. Gitian signatures (in gitian.sigs/${VERSION}-<linux|win|osx>/(your gitian key)/ - -repackage gitian builds for release as stand-alone zip/tar/installer exe - -**Linux .tar.gz:** - - unzip bitcoin-${VERSION}-linux-gitian.zip -d bitcoin-${VERSION}-linux - tar czvf bitcoin-${VERSION}-linux.tar.gz bitcoin-${VERSION}-linux - rm -rf bitcoin-${VERSION}-linux - -**Windows .zip and setup.exe:** - - unzip bitcoin-${VERSION}-win-gitian.zip -d bitcoin-${VERSION}-win - mv bitcoin-${VERSION}-win/bitcoin-*-setup.exe . - zip -r bitcoin-${VERSION}-win.zip bitcoin-${VERSION}-win - rm -rf bitcoin-${VERSION}-win - -**Mac OS X .dmg:** - - mv Bitcoin-Qt.dmg bitcoin-${VERSION}-osx.dmg + 1. source tarball (bitcoin-${VERSION}.tar.gz) + 2. linux 32-bit and 64-bit binaries dist tarballs (bitcoin-${VERSION}-linux[32|64].tar.gz) + 3. windows 32-bit and 64-bit installers and dist zips (bitcoin-${VERSION}-win[32|64]-setup.exe, bitcoin-${VERSION}-win[32|64].zip) + 4. OSX unsigned installer (bitcoin-${VERSION}-osx-unsigned.dmg) + 5. Gitian signatures (in gitian.sigs/${VERSION}-<linux|win|osx-unsigned>/(your gitian key)/ ###Next steps: @@ -104,7 +79,28 @@ Commit your signature to gitian.sigs: pushd gitian.sigs git add ${VERSION}-linux/${SIGNER} git add ${VERSION}-win/${SIGNER} - git add ${VERSION}-osx/${SIGNER} + git add ${VERSION}-osx-unsigned/${SIGNER} + git commit -a + git push # Assuming you can push to the gitian.sigs tree + popd + +Wait for OSX detached signature: + Once the OSX build has 3 matching signatures, Gavin will sign it with the apple App-Store key. + He will then upload a detached signature to be combined with the unsigned app to create a signed binary. + +Create the signed OSX binary: + pushd ./gitian-builder + # Fetch the signature as instructed by Gavin + cp signature.tar.gz inputs/ + ./bin/gbuild -i ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + mv build/out/bitcoin-${VERSION}-osx.dmg ../ + popd + +Commit your signature for the signed OSX binary: + + pushd gitian.sigs + git add ${VERSION}-osx-signed/${SIGNER} git commit -a git push # Assuming you can push to the gitian.sigs tree popd @@ -117,8 +113,6 @@ Commit your signature to gitian.sigs: - Code-sign Windows -setup.exe (in a Windows virtual machine using signtool) - - Code-sign MacOSX .dmg - Note: only Gavin has the code-signing keys currently. - Create `SHA256SUMS.asc` for the builds, and GPG-sign it: diff --git a/qa/rpc-tests/getblocktemplate.py b/qa/rpc-tests/getblocktemplate_longpoll.py index 500662bf87..263a5f6d59 100755 --- a/qa/rpc-tests/getblocktemplate.py +++ b/qa/rpc-tests/getblocktemplate_longpoll.py @@ -3,8 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# Exercise the listtransactions API - from test_framework import BitcoinTestFramework from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from util import * @@ -46,7 +44,7 @@ class LongpollThread(threading.Thread): def run(self): self.node.getblocktemplate({'longpollid':self.longpollid}) -class GetBlockTemplateTest(BitcoinTestFramework): +class GetBlockTemplateLPTest(BitcoinTestFramework): ''' Test longpolling with getblocktemplate. ''' @@ -90,5 +88,5 @@ class GetBlockTemplateTest(BitcoinTestFramework): assert(not thr.is_alive()) if __name__ == '__main__': - GetBlockTemplateTest().main() + GetBlockTemplateLPTest().main() diff --git a/qa/rpc-tests/getblocktemplate_proposals.py b/qa/rpc-tests/getblocktemplate_proposals.py new file mode 100755 index 0000000000..0f7859584a --- /dev/null +++ b/qa/rpc-tests/getblocktemplate_proposals.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework import BitcoinTestFramework +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * + +from binascii import a2b_hex, b2a_hex +from hashlib import sha256 +from struct import pack + + +def check_array_result(object_array, to_match, expected): + """ + Pass in array of JSON objects, a dictionary with key/value pairs + to match against, and another dictionary with expected key/value + pairs. + """ + num_matched = 0 + for item in object_array: + all_match = True + for key,value in to_match.items(): + if item[key] != value: + all_match = False + if not all_match: + continue + for key,value in expected.items(): + if item[key] != value: + raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) + num_matched = num_matched+1 + if num_matched == 0: + raise AssertionError("No objects matched %s"%(str(to_match))) + +def b2x(b): + return b2a_hex(b).decode('ascii') + +# NOTE: This does not work for signed numbers (set the high bit) or zero (use b'\0') +def encodeUNum(n): + s = bytearray(b'\1') + while n > 127: + s[0] += 1 + s.append(n % 256) + n //= 256 + s.append(n) + return bytes(s) + +def varlenEncode(n): + if n < 0xfd: + return pack('<B', n) + if n <= 0xffff: + return b'\xfd' + pack('<H', n) + if n <= 0xffffffff: + return b'\xfe' + pack('<L', n) + return b'\xff' + pack('<Q', n) + +def dblsha(b): + return sha256(sha256(b).digest()).digest() + +def genmrklroot(leaflist): + cur = leaflist + while len(cur) > 1: + n = [] + if len(cur) & 1: + cur.append(cur[-1]) + for i in range(0, len(cur), 2): + n.append(dblsha(cur[i] + cur[i+1])) + cur = n + return cur[0] + +def template_to_bytes(tmpl, txlist): + blkver = pack('<L', tmpl['version']) + mrklroot = genmrklroot(list(dblsha(a) for a in txlist)) + timestamp = pack('<L', tmpl['curtime']) + nonce = b'\0\0\0\0' + blk = blkver + a2b_hex(tmpl['previousblockhash'])[::-1] + mrklroot + timestamp + a2b_hex(tmpl['bits'])[::-1] + nonce + blk += varlenEncode(len(txlist)) + for tx in txlist: + blk += tx + return blk + +def template_to_hex(tmpl, txlist): + return b2x(template_to_bytes(tmpl, txlist)) + +def assert_template(node, tmpl, txlist, expect): + rsp = node.getblocktemplate({'data':template_to_hex(tmpl, txlist),'mode':'proposal'}) + if rsp != expect: + raise AssertionError('unexpected: %s' % (rsp,)) + +class GetBlockTemplateProposalTest(BitcoinTestFramework): + ''' + Test block proposals with getblocktemplate. + ''' + + def run_test(self): + node = self.nodes[0] + tmpl = node.getblocktemplate() + if 'coinbasetxn' not in tmpl: + rawcoinbase = encodeUNum(tmpl['height']) + rawcoinbase += b'\x01-' + hexcoinbase = b2x(rawcoinbase) + hexoutval = b2x(pack('<Q', tmpl['coinbasevalue'])) + tmpl['coinbasetxn'] = {'data': '01000000' + '01' + '0000000000000000000000000000000000000000000000000000000000000000ffffffff' + ('%02x' % (len(rawcoinbase),)) + hexcoinbase + 'fffffffe' + '01' + hexoutval + '00' + '00000000'} + txlist = list(bytearray(a2b_hex(a['data'])) for a in (tmpl['coinbasetxn'],) + tuple(tmpl['transactions'])) + + # Test 0: Capability advertised + assert('proposal' in tmpl['capabilities']) + + # NOTE: This test currently FAILS (regtest mode doesn't enforce block height in coinbase) + ## Test 1: Bad height in coinbase + #txlist[0][4+1+36+1+1] += 1 + #assert_template(node, tmpl, txlist, 'FIXME') + #txlist[0][4+1+36+1+1] -= 1 + + # Test 2: Bad input hash for gen tx + txlist[0][4+1] += 1 + assert_template(node, tmpl, txlist, 'bad-cb-missing') + txlist[0][4+1] -= 1 + + # Test 3: Truncated final tx + lastbyte = txlist[-1].pop() + try: + assert_template(node, tmpl, txlist, 'n/a') + except JSONRPCException: + pass # Expected + txlist[-1].append(lastbyte) + + # Test 4: Add an invalid tx to the end (duplicate of gen tx) + txlist.append(txlist[0]) + assert_template(node, tmpl, txlist, 'bad-txns-duplicate') + txlist.pop() + + # Test 5: Add an invalid tx to the end (non-duplicate) + txlist.append(bytearray(txlist[0])) + txlist[-1][4+1] = b'\xff' + assert_template(node, tmpl, txlist, 'bad-txns-inputs-missingorspent') + txlist.pop() + + # Test 6: Future tx lock time + txlist[0][-4:] = b'\xff\xff\xff\xff' + assert_template(node, tmpl, txlist, 'bad-txns-nonfinal') + txlist[0][-4:] = b'\0\0\0\0' + + # Test 7: Bad tx count + txlist.append(b'') + try: + assert_template(node, tmpl, txlist, 'n/a') + except JSONRPCException: + pass # Expected + txlist.pop() + + # Test 8: Bad bits + realbits = tmpl['bits'] + tmpl['bits'] = '1c0000ff' # impossible in the real world + assert_template(node, tmpl, txlist, 'bad-diffbits') + tmpl['bits'] = realbits + + # Test 9: Bad merkle root + rawtmpl = template_to_bytes(tmpl, txlist) + rawtmpl[4+32] = (rawtmpl[4+32] + 1) % 0x100 + rsp = node.getblocktemplate({'data':b2x(rawtmpl),'mode':'proposal'}) + if rsp != 'bad-txnmrklroot': + raise AssertionError('unexpected: %s' % (rsp,)) + + # Test 10: Bad timestamps + realtime = tmpl['curtime'] + tmpl['curtime'] = 0x7fffffff + assert_template(node, tmpl, txlist, 'time-too-new') + tmpl['curtime'] = 0 + assert_template(node, tmpl, txlist, 'time-too-old') + tmpl['curtime'] = realtime + + # Test 11: Valid block + assert_template(node, tmpl, txlist, None) + + # Test 12: Orphan block + tmpl['previousblockhash'] = 'ff00' * 16 + assert_template(node, tmpl, txlist, 'inconclusive-not-best-prevblk') + +if __name__ == '__main__': + GetBlockTemplateProposalTest().main() diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 9fc661fe80..e3f86d38dc 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -124,6 +124,7 @@ class ReceivedByTest(BitcoinTestFramework): balance_by_account = rec_by_accountArr = self.nodes[1].getreceivedbyaccount(account) txid = self.nodes[0].sendtoaddress(addr, 0.1) + self.sync_all() # listreceivedbyaccount should return received_by_account_json because of 0 confirmations check_array_result(self.nodes[1].listreceivedbyaccount(), diff --git a/src/Makefile.am b/src/Makefile.am index 0d45203c90..3ec9e2f85d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -251,6 +251,7 @@ libbitcoin_common_a_SOURCES = \ # backward-compatibility objects and their sanity checks are linked. libbitcoin_util_a_CPPFLAGS = $(BITCOIN_INCLUDES) libbitcoin_util_a_SOURCES = \ + compat/strnlen.cpp \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ chainparamsbase.cpp \ diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index a2b95d5086..1b638e99e9 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -66,6 +66,20 @@ static bool AppInitRPC(int argc, char* argv[]) // Parameters // ParseParameters(argc, argv); + if (argc<2 || mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) { + std::string strUsage = _("Bitcoin Core RPC client version") + " " + FormatFullVersion() + "\n"; + if (!mapArgs.count("-version")) { + strUsage += "\n" + _("Usage:") + "\n" + + " bitcoin-cli [options] <command> [params] " + _("Send command to Bitcoin Core") + "\n" + + " bitcoin-cli [options] help " + _("List commands") + "\n" + + " bitcoin-cli [options] help <command> " + _("Get help for a command") + "\n"; + + strUsage += "\n" + HelpMessageCli(); + } + + fprintf(stdout, "%s", strUsage.c_str()); + return false; + } if (!boost::filesystem::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", mapArgs["-datadir"].c_str()); return false; @@ -81,20 +95,6 @@ static bool AppInitRPC(int argc, char* argv[]) fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); return false; } - if (argc<2 || mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) { - std::string strUsage = _("Bitcoin Core RPC client version") + " " + FormatFullVersion() + "\n"; - if (!mapArgs.count("-version")) { - strUsage += "\n" + _("Usage:") + "\n" + - " bitcoin-cli [options] <command> [params] " + _("Send command to Bitcoin Core") + "\n" + - " bitcoin-cli [options] help " + _("List commands") + "\n" + - " bitcoin-cli [options] help <command> " + _("Get help for a command") + "\n"; - - strUsage += "\n" + HelpMessageCli(); - } - - fprintf(stdout, "%s", strUsage.c_str()); - return false; - } return true; } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a79e581a80..be7757b0b6 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -59,13 +59,36 @@ bool AppInit(int argc, char* argv[]) boost::thread* detectShutdownThread = NULL; bool fRet = false; + + // + // Parameters + // + // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() + ParseParameters(argc, argv); + + // Process help and version before taking care about datadir + if (mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) + { + std::string strUsage = _("Bitcoin Core Daemon") + " " + _("version") + " " + FormatFullVersion() + "\n"; + + if (mapArgs.count("-version")) + { + strUsage += LicenseInfo(); + } + else + { + strUsage += "\n" + _("Usage:") + "\n" + + " bitcoind [options] " + _("Start Bitcoin Core Daemon") + "\n"; + + strUsage += "\n" + HelpMessage(HMM_BITCOIND); + } + + fprintf(stdout, "%s", strUsage.c_str()); + return false; + } + try { - // - // Parameters - // - // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - ParseParameters(argc, argv); if (!boost::filesystem::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", mapArgs["-datadir"].c_str()); @@ -84,26 +107,6 @@ bool AppInit(int argc, char* argv[]) return false; } - if (mapArgs.count("-?") || mapArgs.count("-help") || mapArgs.count("-version")) - { - std::string strUsage = _("Bitcoin Core Daemon") + " " + _("version") + " " + FormatFullVersion() + "\n"; - - if (mapArgs.count("-version")) - { - strUsage += LicenseInfo(); - } - else - { - strUsage += "\n" + _("Usage:") + "\n" + - " bitcoind [options] " + _("Start Bitcoin Core Daemon") + "\n"; - - strUsage += "\n" + HelpMessage(HMM_BITCOIND); - } - - fprintf(stdout, "%s", strUsage.c_str()); - return false; - } - // Command-line RPC bool fCommandLine = false; for (int i = 1; i < argc; i++) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index e539eb7bd7..8a6a061ea1 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -192,11 +192,6 @@ public: CTestNetParams() { networkID = CBaseChainParams::TESTNET; strNetworkID = "test"; - /** - * The message start string is designed to be unlikely to occur in normal data. - * The characters are rarely used upper ASCII, not valid as UTF-8, and produce - * a large 4-byte int at any alignment. - */ pchMessageStart[0] = 0x0b; pchMessageStart[1] = 0x11; pchMessageStart[2] = 0x09; diff --git a/src/compat.h b/src/compat.h index dade79aae0..dffd4ecf52 100644 --- a/src/compat.h +++ b/src/compat.h @@ -6,6 +6,10 @@ #ifndef BITCOIN_COMPAT_H #define BITCOIN_COMPAT_H +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + #ifdef WIN32 #ifdef _WIN32_WINNT #undef _WIN32_WINNT @@ -84,4 +88,8 @@ typedef u_int SOCKET; #define THREAD_PRIORITY_ABOVE_NORMAL (-2) #endif +#if HAVE_DECL_STRNLEN == 0 +size_t strnlen( const char *start, size_t max_len); +#endif // HAVE_DECL_STRNLEN + #endif // BITCOIN_COMPAT_H diff --git a/src/compat/strnlen.cpp b/src/compat/strnlen.cpp new file mode 100644 index 0000000000..7f3e159887 --- /dev/null +++ b/src/compat/strnlen.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2009-2014 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#include <cstring> + +#if HAVE_DECL_STRNLEN == 0 +size_t strnlen( const char *start, size_t max_len) +{ + const char *end = (const char *)memchr(start, '\0', max_len); + + return end ? (size_t)(end - start) : max_len; +} +#endif // HAVE_DECL_STRNLEN diff --git a/src/core_io.h b/src/core_io.h index b5ed03b8c8..aba1928a36 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -8,6 +8,7 @@ #include <string> #include <vector> +class CBlock; class CScript; class CTransaction; class uint256; @@ -16,6 +17,7 @@ class UniValue; // core_read.cpp extern CScript ParseScript(std::string s); extern bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx); +extern bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); extern uint256 ParseHashUV(const UniValue& v, const std::string& strName); extern std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); diff --git a/src/core_read.cpp b/src/core_read.cpp index d39bc9a780..42e2f8d200 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -4,6 +4,7 @@ #include "core_io.h" +#include "core/block.h" #include "core/transaction.h" #include "script/script.h" #include "serialize.h" @@ -108,6 +109,23 @@ bool DecodeHexTx(CTransaction& tx, const std::string& strHexTx) return true; } +bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) +{ + if (!IsHex(strHexBlk)) + return false; + + std::vector<unsigned char> blockData(ParseHex(strHexBlk)); + CDataStream ssBlock(blockData, SER_NETWORK, PROTOCOL_VERSION); + try { + ssBlock >> block; + } + catch (const std::exception &) { + return false; + } + + return true; +} + uint256 ParseHashUV(const UniValue& v, const string& strName) { string strHex; diff --git a/src/init.cpp b/src/init.cpp index 2d2a05a936..b73c6e8722 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -150,14 +150,9 @@ void Shutdown() { LOCK(cs_main); -#ifdef ENABLE_WALLET - if (pwalletMain) - pwalletMain->SetBestChain(chainActive.GetLocator()); -#endif - if (pblocktree) - pblocktree->Flush(); - if (pcoinsTip) - pcoinsTip->Flush(); + if (pcoinsTip != NULL) { + FlushStateToDisk(); + } delete pcoinsTip; pcoinsTip = NULL; delete pcoinsdbview; @@ -854,10 +849,8 @@ bool AppInit2(boost::thread_group& threadGroup) if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address: '%s'"), mapArgs["-proxy"])); - if (!IsLimited(NET_IPV4)) - SetProxy(NET_IPV4, addrProxy); - if (!IsLimited(NET_IPV6)) - SetProxy(NET_IPV6, addrProxy); + SetProxy(NET_IPV4, addrProxy); + SetProxy(NET_IPV6, addrProxy); SetNameProxy(addrProxy); fProxy = true; } diff --git a/src/key.cpp b/src/key.cpp index 76256b864c..a91ed1cc1d 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -86,6 +86,20 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig) const { return true; } +bool CKey::VerifyPubKey(const CPubKey& pubkey) const { + if (pubkey.IsCompressed() != fCompressed) { + return false; + } + unsigned char rnd[8]; + std::string str = "Bitcoin key verification\n"; + GetRandBytes(rnd, sizeof(rnd)); + uint256 hash; + CHash256().Write((unsigned char*)str.data(), str.size()).Write(rnd, sizeof(rnd)).Finalize((unsigned char*)&hash); + std::vector<unsigned char> vchSig; + Sign(hash, vchSig); + return pubkey.Verify(hash, vchSig); +} + bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) const { if (!fValid) return false; @@ -111,10 +125,7 @@ bool CKey::Load(CPrivKey &privkey, CPubKey &vchPubKey, bool fSkipCheck=false) { if (fSkipCheck) return true; - if (GetPubKey() != vchPubKey) - return false; - - return true; + return VerifyPubKey(vchPubKey); } bool CKey::Derive(CKey& keyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const { @@ -190,5 +201,13 @@ void CExtKey::Decode(const unsigned char code[74]) { } bool ECC_InitSanityCheck() { - return CECKey::SanityCheck(); +#if !defined(USE_SECP256K1) + if (!CECKey::SanityCheck()) { + return false; + } +#endif + CKey key; + key.MakeNewKey(true); + CPubKey pubkey = key.GetPubKey(); + return key.VerifyPubKey(pubkey); } @@ -13,9 +13,10 @@ #include <stdexcept> #include <vector> -struct CExtPubKey; class CPubKey; +struct CExtPubKey; + /** * secp256k1: * const unsigned int PRIVATE_KEY_SIZE = 279; @@ -136,6 +137,12 @@ public: //! Derive BIP32 child key. bool Derive(CKey& keyChild, unsigned char ccChild[32], unsigned int nChild, const unsigned char cc[32]) const; + /** + * Verify thoroughly whether a private key and a public key match. + * This is done using a different mechanism than just regenerating it. + */ + bool VerifyPubKey(const CPubKey& vchPubKey) const; + //! Load private key and check that public key matches. bool Load(CPrivKey& privkey, CPubKey& vchPubKey, bool fSkipCheck); diff --git a/src/main.cpp b/src/main.cpp index 17fa765e8f..88fb31980f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -130,6 +130,12 @@ namespace { // Number of preferrable block download peers. int nPreferredDownload = 0; + + // Dirty block index entries. + set<CBlockIndex*> setDirtyBlockIndex; + + // Dirty block file entries. + set<int> setDirtyFileInfo; } // anon namespace ////////////////////////////////////////////////////////////////////////////// @@ -1137,11 +1143,6 @@ bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos) pos.nPos = (unsigned int)fileOutPos; fileout << block; - // Flush stdio buffers and commit to disk before returning - fflush(fileout.Get()); - if (!IsInitialBlockDownload()) - FileCommit(fileout.Get()); - return true; } @@ -1228,13 +1229,13 @@ void CheckForkWarningConditions() if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (GetBlockProof(*chainActive.Tip()) * 6))) { - if (!fLargeWorkForkFound) + if (!fLargeWorkForkFound && pindexBestForkBase) { std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + pindexBestForkBase->phashBlock->ToString() + std::string("'"); CAlert::Notify(warning, true); } - if (pindexBestForkTip) + if (pindexBestForkTip && pindexBestForkBase) { LogPrintf("CheckForkWarningConditions: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), @@ -1335,7 +1336,7 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state } if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; - pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex)); + setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } @@ -1593,7 +1594,7 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) { AssertLockHeld(cs_main); // Check it again in case a previous version let a bad block in @@ -1732,10 +1733,7 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C } pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); - - CDiskBlockIndex blockindex(pindex); - if (!pblocktree->WriteBlockIndex(blockindex)) - return state.Abort("Failed to write block index"); + setDirtyBlockIndex.insert(pindex); } if (fTxIndex) @@ -1759,10 +1757,23 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C return true; } -// Update the on-disk chain state. -bool static WriteChainState(CValidationState &state, bool forceWrite=false) { +enum FlushStateMode { + FLUSH_STATE_IF_NEEDED, + FLUSH_STATE_PERIODIC, + FLUSH_STATE_ALWAYS +}; + +/** + * Update the on-disk chain state. + * The caches and indexes are flushed if either they're too large, forceWrite is set, or + * fast is not set and it's been a while since the last write. + */ +bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { + LOCK(cs_main); static int64_t nLastWrite = 0; - if (forceWrite || pcoinsTip->GetCacheSize() > nCoinCacheSize || (!IsInitialBlockDownload() && GetTimeMicros() > nLastWrite + 600*1000000)) { + if ((mode == FLUSH_STATE_ALWAYS) || + ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || + (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) { // Typical CCoins structures on disk are around 100 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already @@ -1770,15 +1781,44 @@ bool static WriteChainState(CValidationState &state, bool forceWrite=false) { // overwrite one. Still, use a conservative safety factor of 2. if (!CheckDiskSpace(100 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); + // First make sure all block and undo data is flushed to disk. FlushBlockFile(); + // Then update all block file information (which may refer to block and undo files). + bool fileschanged = false; + for (set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { + if (!pblocktree->WriteBlockFileInfo(*it, vinfoBlockFile[*it])) { + return state.Abort("Failed to write to block index"); + } + fileschanged = true; + setDirtyFileInfo.erase(it++); + } + if (fileschanged && !pblocktree->WriteLastBlockFile(nLastBlockFile)) { + return state.Abort("Failed to write to block index"); + } + for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { + if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(*it))) { + return state.Abort("Failed to write to block index"); + } + setDirtyBlockIndex.erase(it++); + } pblocktree->Sync(); + // Finally flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return state.Abort("Failed to write to coin database"); + // Update best block in wallet (so we can detect restored wallets). + if (mode != FLUSH_STATE_IF_NEEDED) { + g_signals.SetBestChain(chainActive.GetLocator()); + } nLastWrite = GetTimeMicros(); } return true; } +void FlushStateToDisk() { + CValidationState state; + FlushStateToDisk(state, FLUSH_STATE_ALWAYS); +} + // Update chainActive and related internal data structures. void static UpdateTip(CBlockIndex *pindexNew) { chainActive.SetTip(pindexNew); @@ -1837,7 +1877,7 @@ bool static DisconnectTip(CValidationState &state) { } LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. - if (!WriteChainState(state)) + if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; // Resurrect mempool transactions from the disconnected block. BOOST_FOREACH(const CTransaction &tx, block.vtx) { @@ -1900,7 +1940,7 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. - if (!WriteChainState(state)) + if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); @@ -1919,10 +1959,6 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * BOOST_FOREACH(const CTransaction &tx, pblock->vtx) { SyncWithWallets(tx, pblock); } - // Update best block in wallet (so we can detect restored wallets) - // Emit this signal after the SyncWithWallets signals as the wallet relies on that everything up to this point has been synced - if ((chainActive.Height() % 20160) == 0 || ((chainActive.Height() % 144) == 0 && !IsInitialBlockDownload())) - g_signals.SetBestChain(chainActive.GetLocator()); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); @@ -2043,9 +2079,6 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo else CheckForkWarningConditions(); - if (!pblocktree->Flush()) - return state.Abort("Failed to sync block index"); - return true; } @@ -2086,11 +2119,16 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) { if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) pnode->PushInventory(CInv(MSG_BLOCK, hashNewTip)); } - + // Notify external listeners about the new tip. uiInterface.NotifyBlockTip(hashNewTip); } } while(pindexMostWork != chainActive.Tip()); + // Write changes periodically to disk, after relay. + if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) { + return false; + } + return true; } @@ -2123,8 +2161,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) if (pindexBestHeader == NULL || pindexBestHeader->nChainWork < pindexNew->nChainWork) pindexBestHeader = pindexNew; - // Ok if it fails, we'll download the header again next time. - pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew)); + setDirtyBlockIndex.insert(pindexNew); return pindexNew; } @@ -2143,6 +2180,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl LOCK(cs_nBlockSequenceId); pindexNew->nSequenceId = nBlockSequenceId++; } + setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. @@ -2162,15 +2200,11 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl range.first++; mapBlocksUnlinked.erase(it); } - if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex))) - return state.Abort("Failed to write block index"); } } else { if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); } - if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew))) - return state.Abort("Failed to write block index"); } return true; @@ -2178,8 +2212,6 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { - bool fUpdatedLast = false; - LOCK(cs_LastBlockFile); unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; @@ -2195,7 +2227,6 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); } - fUpdatedLast = true; } pos.nFile = nFile; pos.nPos = vinfoBlockFile[nFile].nSize; @@ -2222,11 +2253,7 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd } } - if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, vinfoBlockFile[nFile])) - return state.Abort("Failed to write file info"); - if (fUpdatedLast) - pblocktree->WriteLastBlockFile(nLastBlockFile); - + setDirtyFileInfo.insert(nFile); return true; } @@ -2239,9 +2266,7 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; - if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, vinfoBlockFile[nLastBlockFile])) { - return state.Abort("Failed to write block info"); - } + setDirtyFileInfo.insert(nFile); unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; @@ -2336,6 +2361,73 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo return true; } +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex * const pindexPrev) +{ + uint256 hash = block.GetHash(); + if (hash == Params().HashGenesisBlock()) + return true; + + assert(pindexPrev); + + int nHeight = pindexPrev->nHeight+1; + + // Check proof of work + if ((!Params().SkipProofOfWorkCheck()) && + (block.nBits != GetNextWorkRequired(pindexPrev, &block))) + return state.DoS(100, error("%s : incorrect proof of work", __func__), + REJECT_INVALID, "bad-diffbits"); + + // Check timestamp against prev + if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) + return state.Invalid(error("%s : block's timestamp is too early", __func__), + REJECT_INVALID, "time-too-old"); + + // Check that the block chain matches the known block chain up to a checkpoint + if (!Checkpoints::CheckBlock(nHeight, hash)) + return state.DoS(100, error("%s : rejected by checkpoint lock-in at %d", __func__, nHeight), + REJECT_CHECKPOINT, "checkpoint mismatch"); + + // Don't accept any forks from the main chain prior to last checkpoint + CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) + return state.DoS(100, error("%s : forked chain older than last checkpoint (height %d)", __func__, nHeight)); + + // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: + if (block.nVersion < 2 && + CBlockIndex::IsSuperMajority(2, pindexPrev, Params().RejectBlockOutdatedMajority())) + { + return state.Invalid(error("%s : rejected nVersion=1 block", __func__), + REJECT_OBSOLETE, "bad-version"); + } + + return true; +} + +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) +{ + const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + + // Check that all transactions are finalized + BOOST_FOREACH(const CTransaction& tx, block.vtx) + if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) { + return state.DoS(10, error("%s : contains a non-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); + } + + // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height + // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): + if (block.nVersion >= 2 && + CBlockIndex::IsSuperMajority(2, pindexPrev, Params().EnforceBlockUpgradeMajority())) + { + CScript expect = CScript() << nHeight; + if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || + !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) { + return state.DoS(100, error("%s : block height mismatch in coinbase", __func__), REJECT_INVALID, "bad-cb-height"); + } + } + + return true; +} + bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex) { AssertLockHeld(cs_main); @@ -2358,44 +2450,16 @@ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBloc // Get prev block index CBlockIndex* pindexPrev = NULL; - int nHeight = 0; if (hash != Params().HashGenesisBlock()) { BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) return state.DoS(10, error("%s : prev block not found", __func__), 0, "bad-prevblk"); pindexPrev = (*mi).second; - nHeight = pindexPrev->nHeight+1; - - // Check proof of work - if ((!Params().SkipProofOfWorkCheck()) && - (block.nBits != GetNextWorkRequired(pindexPrev, &block))) - return state.DoS(100, error("%s : incorrect proof of work", __func__), - REJECT_INVALID, "bad-diffbits"); - - // Check timestamp against prev - if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(error("%s : block's timestamp is too early", __func__), - REJECT_INVALID, "time-too-old"); - - // Check that the block chain matches the known block chain up to a checkpoint - if (!Checkpoints::CheckBlock(nHeight, hash)) - return state.DoS(100, error("%s : rejected by checkpoint lock-in at %d", __func__, nHeight), - REJECT_CHECKPOINT, "checkpoint mismatch"); - - // Don't accept any forks from the main chain prior to last checkpoint - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s : forked chain older than last checkpoint (height %d)", __func__, nHeight)); - - // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: - if (block.nVersion < 2 && - CBlockIndex::IsSuperMajority(2, pindexPrev, Params().RejectBlockOutdatedMajority())) - { - return state.Invalid(error("%s : rejected nVersion=1 block", __func__), - REJECT_OBSOLETE, "bad-version"); - } } + if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + return false; + if (pindex == NULL) pindex = AddToBlockIndex(block); @@ -2420,36 +2484,16 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, return true; } - if (!CheckBlock(block, state)) { + if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); } return false; } int nHeight = pindex->nHeight; - // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) - if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) { - pindex->nStatus |= BLOCK_FAILED_VALID; - return state.DoS(10, error("AcceptBlock() : contains a non-final transaction"), - REJECT_INVALID, "bad-txns-nonfinal"); - } - - // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height - // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if (block.nVersion >= 2 && - CBlockIndex::IsSuperMajority(2, pindex->pprev, Params().EnforceBlockUpgradeMajority())) - { - CScript expect = CScript() << nHeight; - if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || - !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) { - pindex->nStatus |= BLOCK_FAILED_VALID; - return state.DoS(100, error("AcceptBlock() : block height mismatch in coinbase"), REJECT_INVALID, "bad-cb-height"); - } - } - // Write block to history file try { unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); @@ -2560,6 +2604,30 @@ bool ProcessNewBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDis return true; } +bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex * const pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +{ + AssertLockHeld(cs_main); + assert(pindexPrev == chainActive.Tip()); + + CCoinsViewCache viewNew(pcoinsTip); + CBlockIndex indexDummy(block); + indexDummy.pprev = pindexPrev; + indexDummy.nHeight = pindexPrev->nHeight + 1; + + // NOTE: CheckBlockHeader is called by CheckBlock + if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + return false; + if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) + return false; + if (!ContextualCheckBlock(block, state, pindexPrev)) + return false; + if (!ConnectBlock(block, state, &indexDummy, viewNew, true)) + return false; + assert(state.IsValid()); + + return true; +} + @@ -3028,7 +3096,7 @@ bool InitBlockIndex() { if (!ActivateBestChain(state, &block)) return error("LoadBlockIndex() : genesis block cannot be activated"); // Force a chainstate write so that when we VerifyDB in a moment, it doesnt check stale data - return WriteChainState(state, true); + return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } catch(std::runtime_error &e) { return error("LoadBlockIndex() : failed to initialize block database: %s", e.what()); } @@ -3221,12 +3289,12 @@ string GetWarnings(string strFor) string strStatusBar; string strRPC; - if (GetBoolArg("-testsafemode", false)) - strRPC = "test"; - if (!CLIENT_VERSION_IS_RELEASE) strStatusBar = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + if (GetBoolArg("-testsafemode", false)) + strStatusBar = strRPC = "testsafemode enabled"; + // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { @@ -4599,11 +4667,6 @@ bool CBlockUndo::WriteToDisk(CDiskBlockPos &pos, const uint256 &hashBlock) hasher << *this; fileout << hasher.GetHash(); - // Flush stdio buffers and commit to disk before returning - fflush(fileout.Get()); - if (!IsInitialBlockDownload()) - FileCommit(fileout.Get()); - return true; } diff --git a/src/main.h b/src/main.h index 1bb0919817..c0d6412528 100644 --- a/src/main.h +++ b/src/main.h @@ -94,6 +94,8 @@ static const unsigned int MAX_HEADERS_RESULTS = 2000; * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning * harder). We'll probably want to make this a per-peer adaptive value at some point. */ static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; +/** Time to wait (in seconds) between writing blockchain state to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 3600; /** "reject" message codes **/ static const unsigned char REJECT_MALFORMED = 0x01; @@ -201,6 +203,8 @@ bool AbortNode(const std::string &msg, const std::string &userMessage=""); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch); +/** Flush all state, indexes and buffers to disk. */ +void FlushStateToDisk(); /** (try to) add transaction to memory pool **/ @@ -457,12 +461,19 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex); bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); // Apply the effects of this block (with given index) on the UTXO set represented by coins -bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool fJustCheck = false); +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool fJustCheck = false); // Context-independent validity checks bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); +// Context-dependent validity checks +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex *pindexPrev); +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex *pindexPrev); + +// Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) +bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex *pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); + // Store block on disk // if dbp is provided, the file is known to already reside on disk bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex **pindex, CDiskBlockPos* dbp = NULL); diff --git a/src/miner.cpp b/src/miner.cpp index 7c3f885410..660173f35b 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -134,6 +134,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) { LOCK2(cs_main, mempool.cs); CBlockIndex* pindexPrev = chainActive.Tip(); + const int nHeight = pindexPrev->nHeight + 1; CCoinsViewCache view(pcoinsTip); // Priority order to process transactions @@ -148,7 +149,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) mi != mempool.mapTx.end(); ++mi) { const CTransaction& tx = mi->second.GetTx(); - if (tx.IsCoinBase() || !IsFinalTx(tx, pindexPrev->nHeight + 1)) + if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight)) continue; COrphan* porphan = NULL; @@ -191,7 +192,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) CAmount nValueIn = coins->vout[txin.prevout.n].nValue; nTotalIn += nValueIn; - int nConf = pindexPrev->nHeight - coins->nHeight + 1; + int nConf = nHeight - coins->nHeight; dPriority += (double)nValueIn * nConf; } @@ -279,7 +280,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) continue; CTxUndo txundo; - UpdateCoins(tx, state, view, txundo, pindexPrev->nHeight+1); + UpdateCoins(tx, state, view, txundo, nHeight); // Added pblock->vtx.push_back(tx); @@ -319,8 +320,8 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) LogPrintf("CreateNewBlock(): total size %u\n", nBlockSize); // Compute final coinbase transaction. - txNew.vout[0].nValue = GetBlockValue(pindexPrev->nHeight+1, nFees); - txNew.vin[0].scriptSig = CScript() << OP_0 << OP_0; + txNew.vout[0].nValue = GetBlockValue(nHeight, nFees); + txNew.vin[0].scriptSig = CScript() << nHeight << OP_0; pblock->vtx[0] = txNew; pblocktemplate->vTxFees[0] = -nFees; @@ -331,13 +332,9 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) pblock->nNonce = 0; pblocktemplate->vTxSigOps[0] = GetLegacySigOpCount(pblock->vtx[0]); - CBlockIndex indexDummy(*pblock); - indexDummy.pprev = pindexPrev; - indexDummy.nHeight = pindexPrev->nHeight + 1; - CCoinsViewCache viewNew(pcoinsTip); CValidationState state; - if (!ConnectBlock(*pblock, state, &indexDummy, viewNew, true)) - throw std::runtime_error("CreateNewBlock() : ConnectBlock failed"); + if (!TestBlockValidity(state, *pblock, pindexPrev, false, false)) + throw std::runtime_error("CreateNewBlock() : TestBlockValidity failed"); } return pblocktemplate.release(); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index a6ecfbcc82..09c79db2d9 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -189,7 +189,7 @@ namespace GUIUtil /* Format a CNodeCombinedStats.dPingTime into a user-readable string or display N/A, if 0*/ QString formatPingTime(double dPingTime); -#ifdef Q_OS_MAC +#if defined(Q_OS_MAC) && QT_VERSION >= 0x050000 // workaround for Qt OSX Bug: // https://bugreports.qt-project.org/browse/QTBUG-15631 // QProgressBar uses around 10% CPU even when app is in background diff --git a/src/rest.cpp b/src/rest.cpp index 9a8793a517..122b361719 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,17 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2012 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying +// Copyright (c) 2009-2014 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <boost/algorithm/string.hpp> -#include "rpcserver.h" -#include "streams.h" -#include "utilstrencodings.h" #include "core/block.h" #include "core/transaction.h" -#include "version.h" #include "main.h" +#include "rpcserver.h" +#include "streams.h" #include "sync.h" +#include "utilstrencodings.h" +#include "version.h" + +#include <boost/algorithm/string.hpp> using namespace std; using namespace json_spirit; @@ -163,7 +164,7 @@ static bool rest_tx(AcceptedConnection *conn, string strJSON = write_string(Value(objTx), false) + "\n"; conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; return true; - } + } } // not reached @@ -203,4 +204,3 @@ bool HTTPReq_REST(AcceptedConnection *conn, conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; return false; } - diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 7919fb1a0c..e8b0f62a83 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -319,7 +319,7 @@ Value gettxoutsetinfo(const Array& params, bool fHelp) Object ret; CCoinsStats stats; - pcoinsTip->Flush(); + FlushStateToDisk(); if (pcoinsTip->GetStats(stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); diff --git a/src/rpcdump.cpp b/src/rpcdump.cpp index c3ffe38cc3..8b95373cff 100644 --- a/src/rpcdump.cpp +++ b/src/rpcdump.cpp @@ -112,6 +112,7 @@ Value importprivkey(const Array& params, bool fHelp) if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); { pwalletMain->MarkDirty(); @@ -253,6 +254,7 @@ Value importwallet(const Array& params, bool fHelp) continue; CKey key = vchSecret.GetKey(); CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); if (pwalletMain->HaveKey(keyid)) { LogPrintf("Skipping import of %s (key already present)\n", CBitcoinAddress(keyid).ToString()); diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index 81120ba981..837a7593b6 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -299,6 +299,25 @@ Value prioritisetransaction(const Array& params, bool fHelp) } +// NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller +static Value BIP22ValidationResult(const CValidationState& state) +{ + if (state.IsValid()) + return Value::null; + + std::string strRejectReason = state.GetRejectReason(); + if (state.IsError()) + throw JSONRPCError(RPC_VERIFY_ERROR, strRejectReason); + if (state.IsInvalid()) + { + if (strRejectReason.empty()) + return "rejected"; + return strRejectReason; + } + // Should be impossible + return "valid?"; +} + Value getblocktemplate(const Array& params, bool fHelp) { if (fHelp || params.size() > 1) @@ -376,6 +395,36 @@ Value getblocktemplate(const Array& params, bool fHelp) else throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); lpval = find_value(oparam, "longpollid"); + + if (strMode == "proposal") + { + const Value& dataval = find_value(oparam, "data"); + if (dataval.type() != str_type) + throw JSONRPCError(RPC_TYPE_ERROR, "Missing data String key for proposal"); + + CBlock block; + if (!DecodeHexBlk(block, dataval.get_str())) + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); + + uint256 hash = block.GetHash(); + BlockMap::iterator mi = mapBlockIndex.find(hash); + if (mi != mapBlockIndex.end()) { + CBlockIndex *pindex = mi->second; + if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) + return "duplicate"; + if (pindex->nStatus & BLOCK_FAILED_MASK) + return "duplicate-invalid"; + return "duplicate-inconclusive"; + } + + CBlockIndex* const pindexPrev = chainActive.Tip(); + // TestBlockValidity only supports blocks built on the current Tip + if (block.hashPrevBlock != pindexPrev->GetBlockHash()) + return "inconclusive-not-best-prevblk"; + CValidationState state; + TestBlockValidity(state, block, pindexPrev, false, true); + return BIP22ValidationResult(state); + } } if (strMode != "template") @@ -478,6 +527,8 @@ Value getblocktemplate(const Array& params, bool fHelp) UpdateTime(pblock, pindexPrev); pblock->nNonce = 0; + static const Array aCaps = boost::assign::list_of("proposal"); + Array transactions; map<uint256, int64_t> setTxIndex; int i = 0; @@ -524,6 +575,7 @@ Value getblocktemplate(const Array& params, bool fHelp) } Object result; + result.push_back(Pair("capabilities", aCaps)); result.push_back(Pair("version", pblock->nVersion)); result.push_back(Pair("previousblockhash", pblock->hashPrevBlock.GetHex())); result.push_back(Pair("transactions", transactions)); @@ -582,41 +634,39 @@ Value submitblock(const Array& params, bool fHelp) + HelpExampleRpc("submitblock", "\"mydata\"") ); - vector<unsigned char> blockData(ParseHex(params[0].get_str())); - CDataStream ssBlock(blockData, SER_NETWORK, PROTOCOL_VERSION); - CBlock pblock; - try { - ssBlock >> pblock; - } - catch (const std::exception &) { + CBlock block; + if (!DecodeHexBlk(block, params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); + + uint256 hash = block.GetHash(); + BlockMap::iterator mi = mapBlockIndex.find(hash); + if (mi != mapBlockIndex.end()) { + CBlockIndex *pindex = mi->second; + if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) + return "duplicate"; + if (pindex->nStatus & BLOCK_FAILED_MASK) + return "duplicate-invalid"; + // Otherwise, we might only have the header - process the block before returning } CValidationState state; - submitblock_StateCatcher sc(pblock.GetHash()); + submitblock_StateCatcher sc(block.GetHash()); RegisterValidationInterface(&sc); - bool fAccepted = ProcessNewBlock(state, NULL, &pblock); + bool fAccepted = ProcessNewBlock(state, NULL, &block); UnregisterValidationInterface(&sc); + if (mi != mapBlockIndex.end()) + { + if (fAccepted && !sc.found) + return "duplicate-inconclusive"; + return "duplicate"; + } if (fAccepted) { if (!sc.found) return "inconclusive"; state = sc.state; } - if (state.IsError()) - { - std::string strRejectReason = state.GetRejectReason(); - throw JSONRPCError(RPC_VERIFY_ERROR, strRejectReason); - } - if (state.IsInvalid()) - { - std::string strRejectReason = state.GetRejectReason(); - if (strRejectReason.empty()) - return "rejected"; - return strRejectReason; - } - - return Value::null; + return BIP22ValidationResult(state); } Value estimatefee(const Array& params, bool fHelp) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 5eda23731d..a10cefcc0b 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -17,13 +17,6 @@ using namespace std; typedef vector<unsigned char> valtype; -static const valtype vchFalse(0); -static const valtype vchZero(0); -static const valtype vchTrue(1, 1); -static const CScriptNum bnZero(0); -static const CScriptNum bnOne(1); -static const CScriptNum bnFalse(0); -static const CScriptNum bnTrue(1); namespace { @@ -239,6 +232,14 @@ bool static CheckMinimalPush(const valtype& data, opcodetype opcode) { bool EvalScript(vector<vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, ScriptError* serror) { + static const CScriptNum bnZero(0); + static const CScriptNum bnOne(1); + static const CScriptNum bnFalse(0); + static const CScriptNum bnTrue(1); + static const valtype vchFalse(0); + static const valtype vchZero(0); + static const valtype vchTrue(1, 1); + CScript::const_iterator pc = script.begin(); CScript::const_iterator pend = script.end(); CScript::const_iterator pbegincodehash = script.begin(); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index b32f3774fe..f9e35e0166 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -82,6 +82,26 @@ BOOST_AUTO_TEST_CASE(key_test1) CPubKey pubkey1C = key1C.GetPubKey(); CPubKey pubkey2C = key2C.GetPubKey(); + BOOST_CHECK(key1.VerifyPubKey(pubkey1)); + BOOST_CHECK(!key1.VerifyPubKey(pubkey1C)); + BOOST_CHECK(!key1.VerifyPubKey(pubkey2)); + BOOST_CHECK(!key1.VerifyPubKey(pubkey2C)); + + BOOST_CHECK(!key1C.VerifyPubKey(pubkey1)); + BOOST_CHECK(key1C.VerifyPubKey(pubkey1C)); + BOOST_CHECK(!key1C.VerifyPubKey(pubkey2)); + BOOST_CHECK(!key1C.VerifyPubKey(pubkey2C)); + + BOOST_CHECK(!key2.VerifyPubKey(pubkey1)); + BOOST_CHECK(!key2.VerifyPubKey(pubkey1C)); + BOOST_CHECK(key2.VerifyPubKey(pubkey2)); + BOOST_CHECK(!key2.VerifyPubKey(pubkey2C)); + + BOOST_CHECK(!key2C.VerifyPubKey(pubkey1)); + BOOST_CHECK(!key2C.VerifyPubKey(pubkey1C)); + BOOST_CHECK(!key2C.VerifyPubKey(pubkey2)); + BOOST_CHECK(key2C.VerifyPubKey(pubkey2C)); + BOOST_CHECK(addr1.Get() == CTxDestination(pubkey1.GetID())); BOOST_CHECK(addr2.Get() == CTxDestination(pubkey2.GetID())); BOOST_CHECK(addr1C.Get() == CTxDestination(pubkey1C.GetID())); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9a31bdf5fd..53c2e7b261 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -57,6 +57,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) uint256 hash; LOCK(cs_main); + Checkpoints::fEnabled = false; // Simple block creation, nothing special yet: BOOST_CHECK(pblocktemplate = CreateNewBlock(scriptPubKey)); @@ -259,6 +260,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) BOOST_FOREACH(CTransaction *tx, txFirst) delete tx; + Checkpoints::fEnabled = true; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index a961b3c5cd..d0062d4544 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -14,13 +14,13 @@ using namespace std; -/** - * safeChars chosen to allow simple messages/URLs/email addresses, but avoid anything - * even possibly remotely dangerous like & or > - */ -static string safeChars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 .,;_/:?@()"); string SanitizeString(const string& str) { + /** + * safeChars chosen to allow simple messages/URLs/email addresses, but avoid anything + * even possibly remotely dangerous like & or > + */ + static string safeChars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 .,;_/:?@()"); string strResult; for (std::string::size_type i = 0; i < str.size(); i++) { diff --git a/src/wallet.cpp b/src/wallet.cpp index 5aea9881cd..000a088b97 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -35,7 +35,7 @@ bool fPayAtLeastCustomFee = true; * Fees smaller than this (in satoshi) are considered zero fee (for transaction creation) * Override with -mintxfee */ -CFeeRate CWallet::minTxFee = CFeeRate(10000); +CFeeRate CWallet::minTxFee = CFeeRate(1000); /** @defgroup mapWallet * @@ -79,6 +79,7 @@ CPubKey CWallet::GenerateNewKey() SetMinVersion(FEATURE_COMPRPUBKEY); CPubKey pubkey = secret.GetPubKey(); + assert(secret.VerifyPubKey(pubkey)); // Create new metadata int64_t nCreationTime = GetTime(); |