From aa9348563c88b984d9c42c41aa5d227145c8ac4b Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Mon, 3 Feb 2014 22:22:07 +0100
Subject: gitian: Make linux build of OpenSSL deterministic

OpenSSL was embedding a timestamp causing its build to be
non-deterministic.
Change deps-linux to be deterministic by using FAKETIME
as needed and disabling it when it gets in the way.
---
 contrib/gitian-descriptors/deps-linux.yml   | 22 ++++++++++++++++++----
 contrib/gitian-descriptors/gitian-linux.yml |  6 +++---
 2 files changed, 21 insertions(+), 7 deletions(-)

(limited to 'contrib')

diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index 12c3179ea1..fc624d8996 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -24,6 +24,9 @@ files:
 script: |
   STAGING="$HOME/install"
   OPTFLAGS='-O2'
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
+  export FAKETIME=$REFERENCE_DATETIME
+  export TZ=UTC
   export LIBRARY_PATH="$STAGING/lib"
   # Integrity Check
   echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3  openssl-1.0.1e.tar.gz"  | sha256sum -c
@@ -37,6 +40,7 @@ script: |
   cd openssl-1.0.1e
   #   need -fPIC to avoid relocation error in 64 bit builds
   ./config no-shared no-zlib no-dso no-krb5 --openssldir=$STAGING -fPIC
+  #   need to build OpenSSL with faketime because a timestamp is embedded into cversion.o
   make
   make install_sw
   cd ..
@@ -48,18 +52,26 @@ script: |
   rm -f $STAGING/lib/libminiupnpc.so* # no way to skip shared lib build
   cd ..
   #
-  tar xjfm qrencode-3.4.3.tar.bz2
+  tar xjf qrencode-3.4.3.tar.bz2
   cd qrencode-3.4.3
+  unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-maintainer-mode --disable-dependency-tracking
+  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking --without-zlib
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
+  export FAKETIME=$REFERENCE_DATETIME
   make $MAKEOPTS install
   cd ..
   #
-  tar xjfm protobuf-2.5.0.tar.bz2
+  tar xjf protobuf-2.5.0.tar.bz2
   cd protobuf-2.5.0
   mkdir -p $STAGING/host/bin
+  unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
   ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
+  export FAKETIME=$REFERENCE_DATETIME
   make $MAKEOPTS install
   cd ..
   #
@@ -67,9 +79,11 @@ script: |
   cd db-4.8.30.NC/build_unix
   #   need --with-pic to avoid relocation error in 64 bit builds
   ../dist/configure --prefix=$STAGING --enable-cxx --disable-shared --with-pic
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
   make $MAKEOPTS library_build
   make install_lib install_include
   cd ../..
   #
   cd $STAGING
-  zip -r $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip include lib bin host
+  find include lib bin host -type f | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 329f4c6b49..7636192800 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -21,8 +21,8 @@ remotes:
 - "url": "https://github.com/bitcoin/bitcoin.git"
   "dir": "bitcoin"
 files:
-- "bitcoin-deps-linux32-gitian-r2.zip"
-- "bitcoin-deps-linux64-gitian-r2.zip"
+- "bitcoin-deps-linux32-gitian-r3.zip"
+- "bitcoin-deps-linux64-gitian-r3.zip"
 - "boost-linux32-1.55.0-gitian-r1.zip"
 - "boost-linux64-1.55.0-gitian-r1.zip"
 script: |
@@ -34,7 +34,7 @@ script: |
   #
   mkdir -p $STAGING
   cd $STAGING
-  unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip
+  unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
   unzip ../build/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
   cd ../build
   #
-- 
cgit v1.2.3


From aabcd11ba69022c6ae24c36d56e107b530fee8ad Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Tue, 4 Feb 2014 11:55:19 +0100
Subject: gitian: Make linux boost dependency completely deterministic

It appears that the output was different every time.

This doesn't affect the final bitcoind/bitcoin-qt, but is confusing
nevertheless.

Fix it by using FAKETIME and zipping files in deterministic order.
---
 contrib/gitian-descriptors/boost-linux.yml | 9 +++++----
 contrib/gitian-descriptors/deps-linux.yml  | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

(limited to 'contrib')

diff --git a/contrib/gitian-descriptors/boost-linux.yml b/contrib/gitian-descriptors/boost-linux.yml
index 48b27aa26d..d752a906a2 100644
--- a/contrib/gitian-descriptors/boost-linux.yml
+++ b/contrib/gitian-descriptors/boost-linux.yml
@@ -20,11 +20,14 @@ files:
 script: |
   STAGING="$HOME/install"
   export LIBRARY_PATH="$STAGING/lib"
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
+  export FAKETIME=$REFERENCE_DATETIME
+  export TZ=UTC
   # Input Integrity Check
   echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52  boost_1_55_0.tar.bz2" | shasum -c
 
   mkdir -p "$STAGING"
-  tar xjf boost_1_55_0.tar.bz2
+  tar --warning=no-timestamp -xjf boost_1_55_0.tar.bz2
   cd boost_1_55_0
   GCCVERSION=$(g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2)
   # note: bjam with -d+2 reveals that -O3 is implied by default, no need to provide it in cxxflags
@@ -38,6 +41,4 @@ script: |
   ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS install
 
   cd "$STAGING"
-  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
-  export FAKETIME=$REFERENCE_DATETIME
-  zip -r $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip *
+  find -type f | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index fc624d8996..8585661458 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -56,7 +56,7 @@ script: |
   cd qrencode-3.4.3
   unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking --without-zlib
+  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking
   # Workaround to prevent re-configuring by make; make all files have a date in the past
   find . -print0 | xargs -r0 touch -t 200001010000
   export FAKETIME=$REFERENCE_DATETIME
@@ -68,7 +68,7 @@ script: |
   mkdir -p $STAGING/host/bin
   unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic
+  ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic --without-zlib
   # Workaround to prevent re-configuring by make; make all files have a date in the past
   find . -print0 | xargs -r0 touch -t 200001010000
   export FAKETIME=$REFERENCE_DATETIME
-- 
cgit v1.2.3


From 1552145ae56bf12f3deaef1cdd8a78b2e923a80e Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Wed, 5 Feb 2014 16:44:58 +0100
Subject: gitian: Sort generated source distribution archive

Sort the filenames in the resulting tar, normalize the time/date
and user/group information.
---
 contrib/gitian-descriptors/gitian-linux.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'contrib')

diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 7636192800..bf355fd828 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -29,6 +29,8 @@ script: |
   STAGING="$HOME/install"
   OPTFLAGS='-O2'
   BINDIR="${OUTDIR}/bin/${GBUILD_BITS}" # 32/64 bit build specific output directory
+  TEMPDIR="$HOME/tempdir"
+  export TZ=UTC
   export LIBRARY_PATH="$STAGING/lib"
   mkdir -p ${BINDIR}
   #
@@ -39,15 +41,20 @@ script: |
   cd ../build
   #
   cd bitcoin
-  export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'`
   ./autogen.sh
   ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt"
   make dist
+  DISTNAME=`echo bitcoin-*.tar.gz`
   mkdir -p distsrc
   cd distsrc
-  tar --strip-components=1 -xf ../bitcoin-*.tar.*
+  tar --strip-components=1 -xf ../$DISTNAME
   ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt"
   make $MAKEOPTS
   make $MAKEOPTS install-strip
+  
+  # sort distribution tar file and normalize user/group/mtime information for deterministic output
   mkdir -p $OUTDIR/src
-  cp ../bitcoin-*.tar.* $OUTDIR/src
+  rm -rf $TEMPDIR
+  mkdir -p $TEMPDIR
+  cd $TEMPDIR
+  tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME
-- 
cgit v1.2.3


From c13a13efec0051e1fe84749c36c6676dc6e267af Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Thu, 6 Feb 2014 11:57:45 +0100
Subject: gitian: add -D flag to ar for deterministic output for linux deps

ar -D: Operate in deterministic mode. When adding files and the archive
index use zero for UIDs, GIDs, timestamps, and use consistent file modes
for all files.  When this option is used, if ar is used with identical
options and identical input files, multiple runs will create identical
output files regardless of the input files' owners, groups, file modes,
or modification times.
---
 contrib/gitian-descriptors/boost-linux.yml | 14 ++++++++++++--
 contrib/gitian-descriptors/deps-linux.yml  | 11 ++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

(limited to 'contrib')

diff --git a/contrib/gitian-descriptors/boost-linux.yml b/contrib/gitian-descriptors/boost-linux.yml
index d752a906a2..a538ff30a8 100644
--- a/contrib/gitian-descriptors/boost-linux.yml
+++ b/contrib/gitian-descriptors/boost-linux.yml
@@ -19,6 +19,7 @@ files:
 - "boost_1_55_0.tar.bz2"
 script: |
   STAGING="$HOME/install"
+  TEMPDIR="$HOME/tmp"
   export LIBRARY_PATH="$STAGING/lib"
   export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
@@ -38,7 +39,16 @@ script: |
 
   ./bootstrap.sh --without-icu
 
-  ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS install
+  ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS -d+2 install
 
+  # post-process all generated libraries to be deterministic
+  # extract them to a temporary directory then re-build them deterministically
+  for LIB in $(find $STAGING -name \*.a); do
+    rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+    ar xv $LIB | cut -b5- > /tmp/list.txt
+    rm $LIB
+    ar crsD $LIB $(cat /tmp/list.txt)
+  done
+  #
   cd "$STAGING"
-  find -type f | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
+  find | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index 8585661458..908f9321a4 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -23,6 +23,7 @@ files:
 - "db-4.8.30.NC.tar.gz"
 script: |
   STAGING="$HOME/install"
+  TEMPDIR="$HOME/tmp"
   OPTFLAGS='-O2'
   export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
@@ -84,6 +85,14 @@ script: |
   make $MAKEOPTS library_build
   make install_lib install_include
   cd ../..
+  # post-process all generated libraries to be deterministic
+  # extract them to a temporary directory then re-build them deterministically
+  for LIB in $(find $STAGING -name \*.a); do
+    rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+    ar xv $LIB | cut -b5- > /tmp/list.txt
+    rm $LIB
+    ar crsD $LIB $(cat /tmp/list.txt)
+  done
   #
   cd $STAGING
-  find include lib bin host -type f | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
+  find include lib bin host | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
-- 
cgit v1.2.3