diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/devtools/README.md | 11 | ||||
| -rwxr-xr-x | contrib/devtools/fix-copyright-headers.py | 53 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/boost-linux.yml | 22 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/boost-win.yml | 19 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/deps-linux.yml | 34 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/deps-win.yml | 15 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/gitian-linux.yml | 20 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/gitian-win.yml | 33 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/protobuf-win.yml | 15 | ||||
| -rw-r--r-- | contrib/gitian-descriptors/qt-win.yml | 41 | ||||
| -rw-r--r-- | contrib/linearize/example-linearize.cfg | 2 | ||||
| -rw-r--r-- | contrib/linearize/linearize.py | 4 |
12 files changed, 216 insertions, 53 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 55d5d24cca..f0d25fd7a2 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -36,3 +36,14 @@ Configuring the github-merge tool for the bitcoin repository is done in the foll git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) git config --global user.signingkey mykeyid (if you want to GPG sign) +## fix-copyright-headers.py + +Every year newly updated files need to have its copyright headers updated to reflect the current year. +If you run this script from src/ it will automatically update the year on the copyright header for all +.cpp and .h files if these have a git commit from the current year. + +For example a file changed in 2014 (with 2014 being the current year): +```// Copyright (c) 2009-2013 The Bitcoin developers``` + +would be changed to: +```// Copyright (c) 2009-2014 The Bitcoin developers```
\ No newline at end of file diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py new file mode 100755 index 0000000000..52fdc99144 --- /dev/null +++ b/contrib/devtools/fix-copyright-headers.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +''' +Run this script inside of src/ and it will look for all the files +that were changed this year that still have the last year in the +copyright headers, and it will fix the headers on that file using +a perl regex one liner. + +For example: if it finds something like this and we're in 2014 + +// Copyright (c) 2009-2013 The Bitcoin developers + +it will change it to + +// Copyright (c) 2009-2014 The Bitcoin developers + +It will do this for all the files in the folder and its children. + +Author: @gubatron +''' +import os +import time + +year = time.gmtime()[0] +last_year = year - 1 +command = "perl -pi -e 's/%s The Bitcoin/%s The Bitcoin/' %s" +listFilesCommand = "find . | grep %s" + +extensions = [".cpp",".h"] + +def getLastGitModifiedDate(filePath): + gitGetLastCommitDateCommand = "git log " + filePath +" | grep Date | head -n 1" + p = os.popen(gitGetLastCommitDateCommand) + result = "" + for l in p: + result = l + break + result = result.replace("\n","") + return result + +n=1 +for extension in extensions: + foundFiles = os.popen(listFilesCommand % extension) + for filePath in foundFiles: + filePath = filePath[1:-1] + if filePath.endswith(extension): + filePath = os.getcwd() + filePath + modifiedTime = getLastGitModifiedDate(filePath) + if len(modifiedTime) > 0 and str(year) in modifiedTime: + print n,"Last Git Modified: ", modifiedTime, " - ", filePath + os.popen(command % (last_year,year,filePath)) + n = n + 1 + + diff --git a/contrib/gitian-descriptors/boost-linux.yml b/contrib/gitian-descriptors/boost-linux.yml index 5d22294dd8..a538ff30a8 100644 --- a/contrib/gitian-descriptors/boost-linux.yml +++ b/contrib/gitian-descriptors/boost-linux.yml @@ -6,6 +6,7 @@ architectures: - "i386" - "amd64" packages: +- "g++" - "unzip" - "pkg-config" - "libtool" @@ -18,12 +19,16 @@ files: - "boost_1_55_0.tar.bz2" script: | STAGING="$HOME/install" + TEMPDIR="$HOME/tmp" export LIBRARY_PATH="$STAGING/lib" + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME + export TZ=UTC # Input Integrity Check echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52 boost_1_55_0.tar.bz2" | shasum -c mkdir -p "$STAGING" - tar xjf boost_1_55_0.tar.bz2 + tar --warning=no-timestamp -xjf boost_1_55_0.tar.bz2 cd boost_1_55_0 GCCVERSION=$(g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2) # note: bjam with -d+2 reveals that -O3 is implied by default, no need to provide it in cxxflags @@ -34,9 +39,16 @@ script: | ./bootstrap.sh --without-icu - ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS install + ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS -d+2 install + # post-process all generated libraries to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $STAGING -name \*.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + ar crsD $LIB $(cat /tmp/list.txt) + done + # cd "$STAGING" - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME - zip -r $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip * + find | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip diff --git a/contrib/gitian-descriptors/boost-win.yml b/contrib/gitian-descriptors/boost-win.yml index 15aeccf543..db5d6bab1d 100644 --- a/contrib/gitian-descriptors/boost-win.yml +++ b/contrib/gitian-descriptors/boost-win.yml @@ -16,8 +16,11 @@ files: - "boost-mingw-gas-cross-compile-2013-03-03.patch" script: | # Defines + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME export TZ=UTC INDIR=$HOME/build + TEMPDIR=$HOME/tmp # Input Integrity Check echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52 boost_1_55_0.tar.bz2" | shasum -c echo "d2b7f6a1d7051faef3c9cf41a92fa3671d905ef1e1da920d07651a43299f6268 boost-mingw-gas-cross-compile-2013-03-03.patch" | shasum -c @@ -35,7 +38,7 @@ script: | mkdir -p $INSTALLPREFIX $BUILDDIR cd $BUILDDIR # - tar xjf $INDIR/boost_1_55_0.tar.bz2 + tar --warning=no-timestamp -xjf $INDIR/boost_1_55_0.tar.bz2 cd boost_1_55_0 GCCVERSION=$($HOST-g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2) echo "using gcc : $GCCVERSION : $HOST-g++ @@ -68,12 +71,16 @@ script: | # http://statmt.org/~s0565741/software/boost_1_52_0/libs/context/doc/html/context/requirements.html # "For cross-compiling the lib you must specify certain additional properties at bjam command line: target-os, abi, binary-format, architecture and address-model." ./bjam toolset=gcc binary-format=pe target-os=windows threadapi=win32 address-model=$BITS threading=multi variant=release link=static runtime-link=static --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 -sNO_ZLIB=1 --layout=tagged --build-type=complete --prefix="$INSTALLPREFIX" $MAKEOPTS install + # post-process all generated libraries to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $INSTALLPREFIX -name \*.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + $HOST-ar crsD $LIB $(cat /tmp/list.txt) + done # cd "$INSTALLPREFIX" - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME - zip -r $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip * - unset LD_PRELOAD - unset FAKETIME + find | sort | zip -X@ $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip done # for BITS in diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml index 41d8d65fb7..908f9321a4 100644 --- a/contrib/gitian-descriptors/deps-linux.yml +++ b/contrib/gitian-descriptors/deps-linux.yml @@ -6,6 +6,7 @@ architectures: - "i386" - "amd64" packages: +- "g++" - "unzip" - "zip" - "pkg-config" @@ -22,7 +23,11 @@ files: - "db-4.8.30.NC.tar.gz" script: | STAGING="$HOME/install" + TEMPDIR="$HOME/tmp" OPTFLAGS='-O2' + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME + export TZ=UTC export LIBRARY_PATH="$STAGING/lib" # Integrity Check echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3 openssl-1.0.1e.tar.gz" | sha256sum -c @@ -36,6 +41,7 @@ script: | cd openssl-1.0.1e # need -fPIC to avoid relocation error in 64 bit builds ./config no-shared no-zlib no-dso no-krb5 --openssldir=$STAGING -fPIC + # need to build OpenSSL with faketime because a timestamp is embedded into cversion.o make make install_sw cd .. @@ -47,18 +53,26 @@ script: | rm -f $STAGING/lib/libminiupnpc.so* # no way to skip shared lib build cd .. # - tar xjfm qrencode-3.4.3.tar.bz2 + tar xjf qrencode-3.4.3.tar.bz2 cd qrencode-3.4.3 + unset FAKETIME # unset fake time during configure, as it does some clock sanity tests # need --with-pic to avoid relocation error in 64 bit builds - ./configure --prefix=$STAGING --enable-static --disable-shared -with-pic --without-tools + ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking + # Workaround to prevent re-configuring by make; make all files have a date in the past + find . -print0 | xargs -r0 touch -t 200001010000 + export FAKETIME=$REFERENCE_DATETIME make $MAKEOPTS install cd .. # - tar xjfm protobuf-2.5.0.tar.bz2 + tar xjf protobuf-2.5.0.tar.bz2 cd protobuf-2.5.0 mkdir -p $STAGING/host/bin + unset FAKETIME # unset fake time during configure, as it does some clock sanity tests # need --with-pic to avoid relocation error in 64 bit builds - ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic + ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic --without-zlib + # Workaround to prevent re-configuring by make; make all files have a date in the past + find . -print0 | xargs -r0 touch -t 200001010000 + export FAKETIME=$REFERENCE_DATETIME make $MAKEOPTS install cd .. # @@ -66,9 +80,19 @@ script: | cd db-4.8.30.NC/build_unix # need --with-pic to avoid relocation error in 64 bit builds ../dist/configure --prefix=$STAGING --enable-cxx --disable-shared --with-pic + # Workaround to prevent re-configuring by make; make all files have a date in the past + find . -print0 | xargs -r0 touch -t 200001010000 make $MAKEOPTS library_build make install_lib install_include cd ../.. + # post-process all generated libraries to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $STAGING -name \*.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + ar crsD $LIB $(cat /tmp/list.txt) + done # cd $STAGING - zip -r $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip include lib bin host + find include lib bin host | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip diff --git a/contrib/gitian-descriptors/deps-win.yml b/contrib/gitian-descriptors/deps-win.yml index ce7acc1e6a..87ebf3e87b 100644 --- a/contrib/gitian-descriptors/deps-win.yml +++ b/contrib/gitian-descriptors/deps-win.yml @@ -26,6 +26,7 @@ script: | export FAKETIME=$REFERENCE_DATETIME export TZ=UTC INDIR=$HOME/build + TEMPDIR=$HOME/tmp # Input Integrity Check echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3 openssl-1.0.1e.tar.gz" | sha256sum -c echo "12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz" | sha256sum -c @@ -107,11 +108,21 @@ script: | # tar xjf $INDIR/qrencode-3.4.3.tar.bz2 cd qrencode-3.4.3 - png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST + png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST --enable-static --disable-shared --without-tools --disable-dependency-tracking + # Workaround to prevent re-configuring by make (resulting in missing m4 error); make all files have a date in the past + find . -print0 | xargs -r0 touch -t 200001010000 make make install cd .. + # post-process all generated libraries to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $INSTALLPREFIX -name \*.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + $HOST-ar crsD $LIB $(cat /tmp/list.txt) + done # cd $INSTALLPREFIX - zip -r $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip include lib + find include lib | sort | zip -X@ $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip done # for BITS in diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 417f31e270..bf355fd828 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -6,6 +6,7 @@ architectures: - "i386" - "amd64" packages: +- "g++" - "libqt4-dev" - "git-core" - "unzip" @@ -20,33 +21,40 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "bitcoin-deps-linux32-gitian-r2.zip" -- "bitcoin-deps-linux64-gitian-r2.zip" +- "bitcoin-deps-linux32-gitian-r3.zip" +- "bitcoin-deps-linux64-gitian-r3.zip" - "boost-linux32-1.55.0-gitian-r1.zip" - "boost-linux64-1.55.0-gitian-r1.zip" script: | STAGING="$HOME/install" OPTFLAGS='-O2' BINDIR="${OUTDIR}/bin/${GBUILD_BITS}" # 32/64 bit build specific output directory + TEMPDIR="$HOME/tempdir" + export TZ=UTC export LIBRARY_PATH="$STAGING/lib" mkdir -p ${BINDIR} # mkdir -p $STAGING cd $STAGING - unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip + unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip unzip ../build/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip cd ../build # cd bitcoin - export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'` ./autogen.sh ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt" make dist + DISTNAME=`echo bitcoin-*.tar.gz` mkdir -p distsrc cd distsrc - tar --strip-components=1 -xf ../bitcoin-*.tar.* + tar --strip-components=1 -xf ../$DISTNAME ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt" make $MAKEOPTS make $MAKEOPTS install-strip + + # sort distribution tar file and normalize user/group/mtime information for deterministic output mkdir -p $OUTDIR/src - cp ../bitcoin-*.tar.* $OUTDIR/src + rm -rf $TEMPDIR + mkdir -p $TEMPDIR + cd $TEMPDIR + tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 9364db4ef7..6e43c21823 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -22,8 +22,8 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "qt-win32-5.2.0-gitian-r1.zip" -- "qt-win64-5.2.0-gitian-r1.zip" +- "qt-win32-5.2.0-gitian-r2.zip" +- "qt-win64-5.2.0-gitian-r2.zip" - "boost-win32-1.55.0-gitian-r6.zip" - "boost-win64-1.55.0-gitian-r6.zip" - "bitcoin-deps-win32-gitian-r10.zip" @@ -35,7 +35,16 @@ script: | export TZ=UTC INDIR=$HOME/build OPTFLAGS='-O2' + TEMPDIR="$HOME/tempdir" NEEDDIST=1 + # Qt: workaround for determinism in resource ordering + # Qt5's rcc uses a QHash to store the files for the resource. + # A security fix in QHash makes the ordering of keys to be different on every run + # (https://qt.gitorious.org/qt/qtbase/commit/c01eaa438200edc9a3bbcd8ae1e8ded058bea268). + # This is good in general but qrc shouldn't be doing a traversal over a randomized container. + # The thorough solution would be to use QMap instead of QHash, but this requires patching Qt. + # For now luckily there is a test mode that forces a fixed seed. + export QT_RCC_TEST=1 for BITS in 32 64; do # for architectures # STAGING=$HOME/staging${BITS} @@ -46,31 +55,28 @@ script: | else HOST=x86_64-w64-mingw32 fi + export PATH=$STAGING/host/bin:$PATH mkdir -p $STAGING $BUILDDIR $BINDIR # cd $STAGING - unzip $INDIR/qt-win${BITS}-5.2.0-gitian-r1.zip + unzip $INDIR/qt-win${BITS}-5.2.0-gitian-r2.zip unzip $INDIR/boost-win${BITS}-1.55.0-gitian-r6.zip unzip $INDIR/bitcoin-deps-win${BITS}-gitian-r10.zip unzip $INDIR/protobuf-win${BITS}-2.5.0-gitian-r4.zip if [ "$NEEDDIST" == "1" ]; then # Make source code archive which is architecture independent so it only needs to be done once - cd $HOME/build/ - cd bitcoin - export PATH=$STAGING/host/bin:$PATH - export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'` + cd $HOME/build/bitcoin ./autogen.sh ./configure --bindir=$OUTDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" make dist - mkdir -p $OUTDIR/src - cp -f bitcoin-*.tar.* $OUTDIR/src + DISTNAME=`echo bitcoin-*.tar.gz` NEEDDIST=0 fi # Build platform-dependent executables from source archive cd $BUILDDIR mkdir -p distsrc cd distsrc - tar --strip-components=1 -xf $HOME/build/bitcoin/bitcoin-*.tar.* + tar --strip-components=1 -xf $HOME/build/bitcoin/$DISTNAME ./configure --bindir=$BINDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME @@ -82,3 +88,10 @@ script: | unset FAKETIME done # for BITS in + # sort distribution tar file and normalize user/group/mtime information for deterministic output + mkdir -p $OUTDIR/src + rm -rf $TEMPDIR + mkdir -p $TEMPDIR + cd $TEMPDIR + tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME + diff --git a/contrib/gitian-descriptors/protobuf-win.yml b/contrib/gitian-descriptors/protobuf-win.yml index 7fcac72fea..543f20b394 100644 --- a/contrib/gitian-descriptors/protobuf-win.yml +++ b/contrib/gitian-descriptors/protobuf-win.yml @@ -17,6 +17,7 @@ script: | # export TZ=UTC INDIR=$HOME/build + TEMPDIR=$HOME/tmp OPTFLAGS="-O2" # Integrity Check echo "13bfc5ae543cf3aa180ac2485c0bc89495e3ae711fc6fab4f8ffe90dfb4bb677 protobuf-2.5.0.tar.bz2" | sha256sum -c @@ -44,13 +45,21 @@ script: | # Now recompile with the mingw cross-compiler: make distclean ./configure --prefix=$INSTALLPREFIX --enable-shared=no --disable-dependency-tracking --with-protoc=$INSTALLPREFIX/host/bin/protoc --host=$HOST CXXFLAGS="-frandom-seed=11 ${OPTFLAGS}" + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME make make install + # post-process all generated libraries to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $INSTALLPREFIX -name \*.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + $HOST-ar crsD $LIB $(cat /tmp/list.txt) + done # cd $INSTALLPREFIX - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME - zip -r $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip include lib host + find include lib host | sort | zip -X@ $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip unset LD_PRELOAD unset FAKETIME done # for BITS in diff --git a/contrib/gitian-descriptors/qt-win.yml b/contrib/gitian-descriptors/qt-win.yml index 3b9aeeb19f..e3de2c02ef 100644 --- a/contrib/gitian-descriptors/qt-win.yml +++ b/contrib/gitian-descriptors/qt-win.yml @@ -21,6 +21,15 @@ script: | # Defines export TZ=UTC INDIR=$HOME/build + TEMPDIR=$HOME/tmp + # Qt: workaround for determinism in resource ordering + # Qt5's rcc uses a QHash to store the files for the resource. + # A security fix in QHash makes the ordering of keys to be different on every run + # (https://qt.gitorious.org/qt/qtbase/commit/c01eaa438200edc9a3bbcd8ae1e8ded058bea268). + # This is good in general but qrc shouldn't be doing a traversal over a randomized container. + # The thorough solution would be to use QMap instead of QHash, but this requires patching Qt. + # For now luckily there is a test mode that forces a fixed seed. + export QT_RCC_TEST=1 # Integrity Check echo "395ec72277c5786c65b8163ef5817fd03d0a1f524a6d47f53624baf8056f1081 qt-everywhere-opensource-src-5.2.0.tar.gz" | sha256sum -c @@ -47,31 +56,37 @@ script: | cd qt-everywhere-opensource-src-5.2.0 SPECNAME="win32-g++" SPECFILE="qtbase/mkspecs/${SPECNAME}/qmake.conf" - sed 's/$TODAY/2011-01-30/' -i configure + sed 's/qt_instdate=`date +%Y-%m-%d`/qt_instdate=2011-01-30/' -i qtbase/configure sed --posix "s|QMAKE_CFLAGS = -pipe -fno-keep-inline-dllexport|QMAKE_CFLAGS\t\t= -pipe -fno-keep-inline-dllexport -isystem /usr/$HOST/include/ -frandom-seed=qtbuild -I$DEPSDIR/include|" -i ${SPECFILE} sed --posix "s|QMAKE_LFLAGS =|QMAKE_LFLAGS\t\t= -L$DEPSDIR/lib|" -i ${SPECFILE} - # ar adds timestamps to every object file included in the static library - # providing -D as ar argument is supposed to solve it, but doesn't work as qmake strips off the arguments and adds -M to pass a script... - # which somehow cannot be combined with other flags. - # use faketime only for ar, as it confuses make/qmake into hanging sometimes - sed --posix "s|QMAKE_LIB = \\\$\\\${CROSS_COMPILE}ar -ru|QMAKE_LIB\t\t= $HOME/ar -Dr|" -i ${SPECFILE} - echo '#!/bin/bash' > $HOME/ar - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> $HOME/ar - echo "$HOST-ar \"\$@\"" >> $HOME/ar - chmod +x $HOME/ar + # Before we tried to pass arguments to ar (static linking) in using QMAKE_LIB, however + # qt removes the arguments for ar and provides a script which makes it impossible to pass the determinism flag - + # so rather than try to replace ar, post-process all libraries and plugins at the end. + # # Don't load faketime while compiling Qt, qmake will get stuck in nearly infinite loops #export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME + #export FAKETIME=$REFERENCE_DATETIME # # Compile static libraries, and use statically linked openssl (-openssl-linked): OPENSSL_LIBS="-L$DEPSDIR/lib -lssl -lcrypto -lgdi32" ./configure -prefix $INSTALLPREFIX -bindir $INSTALLPREFIX/host/bin -confirm-license -release -opensource -static -xplatform $SPECNAME -device-option CROSS_COMPILE="$HOST-" -no-audio-backend -no-javascript-jit -no-sql-sqlite -no-sql-odbc -no-nis -no-cups -no-iconv -no-dbus -no-gif -no-opengl -no-compile-examples -no-feature-style-windowsce -no-feature-style-windowsmobile -no-qml-debug -openssl-linked -skip qtsvg -skip qtwebkit -skip qtwebkit-examples -skip qtserialport -skip qtdeclarative -skip qtmultimedia -skip qtimageformats -skip qtlocation -skip qtsensors -skip qtquick1 -skip qtquickcontrols -skip qtactiveqt -skip qtconnectivity -skip qtwinextras -skip qtxmlpatterns -skip qtscript -skip qtdoc -system-libpng -system-zlib make $MAKEOPTS install + # post-process all generated libraries and plugins to be deterministic + # extract them to a temporary directory then re-build them deterministically + for LIB in $(find $INSTALLPREFIX -name *.a); do + rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR + $HOST-ar xv $LIB | cut -b5- > /tmp/list.txt + rm $LIB + $HOST-ar crsD $LIB $(cat /tmp/list.txt) + done # cd $INSTALLPREFIX - + # Remove unused non-deterministic stuff + rm host/bin/qtpaths.exe lib/libQt5Bootstrap.a lib/libQt5Bootstrap.la # as zip stores file timestamps, use faketime to intercept stat calls to set dates for all files to reference date export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - zip -r $OUTDIR/qt-win${BITS}-5.2.0-gitian-r1.zip * + export FAKETIME=$REFERENCE_DATETIME + find -print0 | xargs -r0 touch # fix up timestamps before packaging + find | sort | zip -X@ $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip unset LD_PRELOAD unset FAKETIME done # for BITS in diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 65bba712a5..f5cdab5325 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -1,7 +1,7 @@ # bitcoind RPC settings rpcuser=someuser -rpcpass=somepassword +rpcpassword=somepassword host=127.0.0.1 port=8332 diff --git a/contrib/linearize/linearize.py b/contrib/linearize/linearize.py index ea3814ab72..650f7d3684 100644 --- a/contrib/linearize/linearize.py +++ b/contrib/linearize/linearize.py @@ -71,7 +71,7 @@ def getblock(rpc, settings, n): def get_blocks(settings): rpc = BitcoinRPC(settings['host'], settings['port'], - settings['rpcuser'], settings['rpcpass']) + settings['rpcuser'], settings['rpcpassword']) outf = open(settings['output'], 'ab') @@ -118,7 +118,7 @@ if __name__ == '__main__': settings['min_height'] = 0 if 'max_height' not in settings: settings['max_height'] = 279000 - if 'rpcuser' not in settings or 'rpcpass' not in settings: + if 'rpcuser' not in settings or 'rpcpassword' not in settings: print "Missing username and/or password in cfg file" sys.exit(1) |