From 570d769c6c59b9f6d1a2b95b2ed60432cb33b3ba Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 16 Jan 2020 14:40:48 -0500 Subject: guix: Build support for Windows --- contrib/guix/guix-build.sh | 40 +++++---- contrib/guix/libexec/build.sh | 191 ++++++++++++++++++++++++++++++++---------- contrib/guix/manifest.scm | 117 +++++++++++++++++--------- 3 files changed, 244 insertions(+), 104 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 2daa8aba5e..e7568f0561 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -20,26 +20,32 @@ time-machine() { } # Deterministically build Bitcoin Core for HOSTs (overriable by environment) -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do +for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm' - # shellcheck disable=SC2086 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # shellcheck disable=SC2086 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) done diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ee207a957c..f74d32fe71 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -36,23 +36,41 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Determine output paths to use in CROSS_* environment variables -CROSS_GLIBC="$(store_path glibc-cross-${HOST})" -CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" -CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" -CROSS_GCC="$(store_path gcc-cross-${HOST})" -CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... -CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # Set environment variables to point Guix's cross-toolchain to the right # includes/libs for $HOST -# -# NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because -# the limits.h in it is missing a '#include_next ' -# -export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" -export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" -export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" + export CPATH="${NATIVE_GCC}/include" + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *linux*) + CROSS_GLIBC="$(store_path glibc-cross-${HOST})" + CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" + CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" + CROSS_GCC="$(store_path gcc-cross-${HOST})" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because + # the limits.h in it is missing a '#include_next ' + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" @@ -74,16 +92,20 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes [ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env # Determine the correct value for -Wl,--dynamic-linker for the current $HOST -glibc_dynamic_linker=$( - case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - *) exit 1 ;; - esac -) +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + *) exit 1 ;; + esac + ) + ;; +esac # Environment variables for determinism export QT_RCC_TEST=1 @@ -111,7 +133,8 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + mingw32_CFLAGS='-DMINGW_HAS_SECURE_API=1 -pipe' ########################### @@ -136,11 +159,26 @@ DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" # Binary Tarball Building # ########################### -# Similar flags to Gitian -CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" -HOST_CFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_CXXFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +case "$HOST" in + *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; +esac + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" ;; +esac # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" @@ -160,7 +198,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" ${CONFIGFLAGS} \ CFLAGS="${HOST_CFLAGS}" \ CXXFLAGS="${HOST_CXXFLAGS}" \ - LDFLAGS="${HOST_LDFLAGS}" + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -169,9 +207,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} + + case "$HOST" in + *linux*) + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} + ;; + esac + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} + ;; + esac # Setup the directory where our Bitcoin Core build for HOST will be # installed. This directory will also later serve as the input for our @@ -180,9 +230,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + + case "$HOST" in + *mingw*) + cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe + ;; + esac ( cd installed + case "$HOST" in + *mingw*) + mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll + ;; + esac + # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete @@ -196,19 +258,56 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + ;; + esac # Finally, deterministically produce {non-,}debug binary tarballs ready # for release - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + esac ) ) + +case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir unsigned + cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; +esac diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c25ac2977b..1e2d3507d3 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -10,11 +10,14 @@ (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages installers) (gnu packages linux) + (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (guix build-system gnu) (guix build-system trivial) (guix gexp) (guix packages) @@ -102,45 +105,77 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc gcc) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + + (packages->manifest - (list ;; The Basics - bash-minimal - which - coreutils - util-linux - ;; File(system) inspection - file - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - ;; Compression and archiving - tar - bzip2 - gzip - xz - zlib - ;; Build tools - gnu-make - libtool - autoconf - automake - pkg-config - ;; Scripting - perl - python-3.7 - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27) - ;; Cross gcc 9 toolchains targeting glibc 2.27 - (make-bitcoin-cross-toolchain "i686-linux-gnu") - (make-bitcoin-cross-toolchain "x86_64-linux-gnu") - (make-bitcoin-cross-toolchain "aarch64-linux-gnu") - (make-bitcoin-cross-toolchain "arm-linux-gnueabihf") - ;; The glibc 2.27 for riscv64 needs gcc 7 to successfully build (see: - ;; https://www.gnu.org/software/gcc/gcc-7/changes.html#riscv). The final - ;; toolchain is still a gcc 9 toolchain targeting glibc 2.27. - (make-bitcoin-cross-toolchain "riscv64-linux-gnu" - #:base-gcc-for-libc gcc-7))) + (append + (list ;; The Basics + bash-minimal + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + ;; Scripting + perl + python-3.7 + ;; Native gcc 9 toolchain targeting glibc 2.27 + (make-gcc-toolchain gcc-9 glibc-2.27)) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) + ((string-contains target "riscv64-linux-") + (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" + #:base-gcc-for-libc gcc-7))) + ((string-contains target "-linux-") + (list (make-bitcoin-cross-toolchain target))) + (else '()))))) -- cgit v1.2.3 From 91897c95e191d293eb27d8af15cbeafc5b8f3895 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 16 Jan 2020 14:42:48 -0500 Subject: guix: Improve guix-build.sh documentation --- contrib/guix/guix-build.sh | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index e7568f0561..043ab69060 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,6 +13,8 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ @@ -32,6 +34,53 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irrepreducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # # shellcheck disable=SC2086 time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ -- cgit v1.2.3 From df953a4c9a6143f45864757b706c88b6fa70545a Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 31 Jan 2020 08:26:25 -0500 Subject: guix: Appease shellcheck. --- contrib/guix/guix-build.sh | 1 + contrib/guix/libexec/build.sh | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 043ab69060..5e794e8063 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -22,6 +22,7 @@ time-machine() { } # Deterministically build Bitcoin Core for HOSTs (overriable by environment) +# shellcheck disable=SC2153 for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index f74d32fe71..bb3917b860 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -55,10 +55,10 @@ case "$HOST" in export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" ;; *linux*) - CROSS_GLIBC="$(store_path glibc-cross-${HOST})" - CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" - CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" - CROSS_GCC="$(store_path gcc-cross-${HOST})" + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) -- cgit v1.2.3 From c4cce00eac691625b78b92f7dba0b7f57def19e5 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 31 Jan 2020 08:28:23 -0500 Subject: guix: Remove dead links from README. --- contrib/guix/README.md | 2 -- 1 file changed, 2 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 8500379025..9f99b36f88 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -220,8 +220,6 @@ repository and will likely put one up soon. [guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html [guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html -[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 -- cgit v1.2.3 From acf4b3b3b5accf60a19441a0298ef27001b78e72 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 13 Feb 2020 15:52:44 -0500 Subject: guix: Make x86_64-w64-mingw32 builds reproducible - Add "--no-insert-timestamp" LDFLAG for x86_64-w64-mingw32 builds "The option --no-insert-timestamp can be used to insert a zero value for the timestamp, this ensuring that binaries produced from identical sources will compare identically." - ld(1) - Set "SetDateSave off" in NSIS script From https://nsis.sourceforge.io/Docs/Chapter4.html#flags "This command sets the file date/time saving flag which is used by the File command to determine whether or not to save the last write date and time of the file, so that it can be restored on installation. Valid flags are 'on' and 'off'. 'on' is the default." - Add commented out NSIS options for reproducibility debugging in NSIS script - Make ZIPs deterministic by reseting file modification times to SOURCE_DATE_EPOCH using touch(1) (Reference: https://reproducible-builds.org/docs/archives/) --- contrib/guix/libexec/build.sh | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bb3917b860..a970eaa625 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -178,6 +178,7 @@ HOST_CXXFLAGS="$HOST_CFLAGS" # LDFLAGS case "$HOST" in *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac # Make $HOST-specific native binaries from depends available in $PATH @@ -271,10 +272,14 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # for release case "$HOST" in *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" find "${DISTNAME}" -not -name "*.dbg" \ | sort \ | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" find "${DISTNAME}" -name "*.dbg" \ | sort \ | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ -- cgit v1.2.3 From ef4f7e4c45c60a69406134122f091c77c6ef740f Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Sat, 14 Mar 2020 09:42:02 -0400 Subject: guix: Set the well-known timezone env var --- contrib/guix/libexec/build.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index a970eaa625..5d09e6b510 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +export TZ=UTC export LC_ALL=C set -e -o pipefail -- cgit v1.2.3 From 93e41b7e3b54c17fd1b4c61ee95fc0dc2827e954 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 24 Mar 2020 14:08:56 -0400 Subject: guix: Use gcc-8 for mingw-w64 instead of 7 We're using mingw-w64 6.0.0, which is paired with gcc-8 in most distros. --- contrib/guix/manifest.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 1e2d3507d3..4e99a792cd 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -114,7 +114,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-ssp-fixed-gcc gcc) + #:xgcc (make-ssp-fixed-gcc gcc-8) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and -- cgit v1.2.3 From 360a9e0ad50a36ec79a1a160dbed3966689fd41c Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 24 Mar 2020 15:40:33 -0400 Subject: guix: Bump time-machine for mingw-w64 patches This bump will includes a couple of commits which improve the reproducibility of the mingw-w64 toolchain. Most of which came from debian. They will be upstreamed as upstream Guix release timeline allows. --- contrib/guix/guix-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 5e794e8063..1a6fe66151 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -17,7 +17,7 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" # across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + --commit=cf347affd642c709f7e423f7111794ffdb09f5a6 \ -- "$@" } -- cgit v1.2.3 From ff821dd2a1c600488d11e7d9a20e9179ecc9144b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 25 Mar 2020 17:17:14 -0400 Subject: guix: Reinstate make-ssp-fixed-gcc Unfortunately, gcc is still not smart enough to detect whether or not mingw-w64 provides ssp, so let's put it back just for mingw-w64. --- contrib/guix/manifest.scm | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 4e99a792cd..c698bf0763 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -24,6 +24,17 @@ (guix profiles) (guix utils)) +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + (define (make-gcc-rpath-link xgcc) "Given a XGCC package, return a modified package that replace each instance of -rpath in the default system spec that's inserted by Guix with -rpath-link" -- cgit v1.2.3 From 3f1f03c67a8e9edf487f08d272adb18b0a3942c8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 30 Mar 2020 13:40:16 -0400 Subject: guix: Spelling fixes --- contrib/guix/guix-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 1a6fe66151..1505ad9657 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -21,7 +21,7 @@ time-machine() { -- "$@" } -# Deterministically build Bitcoin Core for HOSTs (overriable by environment) +# Deterministically build Bitcoin Core for HOSTs (overridable by environment) # shellcheck disable=SC2153 for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do @@ -67,7 +67,7 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # into. # # While we don't want to map our current working directory to the - # same exact path (as this introduces irrepreducibility), we do want + # same exact path (as this introduces irreproducibility), we do want # it to be at a _fixed_ path _somewhere_ inside the isolated # container so that we have something to build. '/bitcoin' was # chosen arbitrarily. -- cgit v1.2.3 From 449d8fe25bbe25daacfc67aa89ca32b0a3254c5a Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 30 Mar 2020 13:41:11 -0400 Subject: guix: Expand on INT trap message --- contrib/guix/guix-build.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 1505ad9657..ca79d0aceb 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -21,12 +21,29 @@ time-machine() { -- "$@" } +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + # Deterministically build Bitcoin Core for HOSTs (overridable by environment) # shellcheck disable=SC2153 for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build - trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT + trap 'int_trap ${host}' INT ( # Required for 'contrib/guix/manifest.scm' to output the right manifest -- cgit v1.2.3 From 35a96792dda9e78165b1598aeac7b2ab759e7be5 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 30 Mar 2020 13:41:34 -0400 Subject: guix: Check mingw symbols, improve SSP fix docs --- contrib/guix/libexec/build.sh | 2 +- contrib/guix/manifest.scm | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 5d09e6b510..94ff9c273d 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -211,7 +211,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" make -C src --jobs=1 check-security ${V:+V=1} case "$HOST" in - *linux*) + *linux*|*mingw*) # Check that executables only contain allowed gcc, glibc and libstdc++ # version symbols for Linux distro back-compatibility. make -C src --jobs=1 check-symbols ${V:+V=1} diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c698bf0763..c9d7193c85 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -26,7 +26,10 @@ (define (make-ssp-fixed-gcc xgcc) "Given a XGCC package, return a modified package that uses the SSP function -from glibc instead of from libssp.so. Taken from: +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (package (inherit xgcc) -- cgit v1.2.3 From 93439a71eda49fb69f1e82966a23a946733aa6fa Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 7 Apr 2020 18:28:28 -0400 Subject: guix: Bump to upstream commit with mingw-w64 changes Most of the mingw-w64 toolchain changes have now been upstreamed, we can point to a commit that exists upstream. NOTE: I'm not changing the URL yet until we see that Guix upstream will accept all my patches for macOS. ----- The Guix tree that's referred to by this commit contains the following changes relevant to our mingw-w64 build: b066c25026 Adds a PACKAGES-WITH-*PATCHES procedure which we can use in the future to apply patches to packages if those patches are not considered appropriate to upstream Guix 4719b71572 Adds mingw-w64 (the libc itself) reproducibility patches, taken from debian. 79825bee07 + 401d28e433 + c1c50cb5b0 Add mingw-w64 specific binutils patches, taken from debian. Specifically, the "Make DLL import libraries reproducible" patch made libbitcoinconsensus.dll.a build reproducibly. The followup commits were hotfixes for my mistakes. 0f864175dc Bumps mingw-w64 to v7.0.0. This is the first release that enables secure APIs by default (which we need), and gains _FORTIFY_SOURCE support. This will also be what Ubuntu Focal 20.04 LTS releases with. cdf00cf75d Bumps NSIS to v3.05. This is the first release that includes a fix for a reproducibility bug found by some of the electrum developers. See details here: https://sourceforge.net/p/nsis/bugs/1230/ --- contrib/guix/guix-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index ca79d0aceb..e20b2a048d 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -17,7 +17,7 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" # across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=cf347affd642c709f7e423f7111794ffdb09f5a6 \ + --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ -- "$@" } -- cgit v1.2.3 From ba0b99bdd613ba7f17c6247ece3001e1b44759a3 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 7 Apr 2020 18:42:01 -0400 Subject: guix: Don't set MINGW_HAS_SECURE_API CFLAG in depends This is no longer needed after 3bef7c22 in the mingw-w64 git repository, which is first included in mingw-w64 v7.0.0. As of the previous bump to our Guix time machine, we now use mingw-w64 v7.0.0. --- contrib/guix/libexec/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 94ff9c273d..63feb501f2 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -134,8 +134,7 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ - mingw32_CFLAGS='-DMINGW_HAS_SECURE_API=1 -pipe' + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' ########################### -- cgit v1.2.3 From 0b66d22da5f53640e22f05adf880782c613e6d0f Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 7 Apr 2020 18:23:53 -0400 Subject: guix: Use gcc-9 for mingw-w64 instead of 8 The libtool unsorted 'find' determinism issue seemed to have been solved in gcc-9's git: d41cd173e23ebea7c758644d6ad6e0fde1c2e3a6 or SVN: r262451 Furthermore, it seems that Ubuntu Focal 20.04 LTS is going to ship with gcc 9 and mingw-w64 7, which will match what we have now. ----- A note on this: Careful observers will see that previously I stated that all released versions of gcc were bootstrapped with a libtool 2.2.7a, meaning that they all had the unsorted 'find' determinism issue first resolved in libtool 2.2.7b. However, I was mistaken, gcc's ltmain.sh CLAIMS it was generated by libtool 2.2.7a, but it was in fact edited manually. It seems that gcc maintains their own versions of ltmain.sh and libtool.m4, and only sometimes backports patches from upstream. Quite confusing. --- contrib/guix/manifest.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c9d7193c85..86c1a8d27f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -128,7 +128,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-ssp-fixed-gcc gcc-8) + #:xgcc (make-ssp-fixed-gcc gcc-9) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and -- cgit v1.2.3 From a35e3235891d35daa167116cc70340140e883f06 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 7 Apr 2020 19:27:31 -0400 Subject: guix: Appease travis. --- contrib/guix/libexec/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 63feb501f2..6ef803340b 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -export TZ=UTC export LC_ALL=C set -e -o pipefail +export TZ=UTC # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." -- cgit v1.2.3