From d5a71e97853ea9e1b879e8c76bfb01d4bef33172 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 22 Feb 2021 17:04:19 -0500 Subject: guix: Use --cores instead of --max-jobs In Guix, there are two flags for controlling parallelism: Note: When I say "derivation," think "package" --cores=n - controls the number of CPU cores to build each derivation. This is the value passed to `make`'s `--jobs=` flag. - defaults to 0: as many cores as is available --max-jobs=n - controls how many derivations can be built in parallel - defaults to 1 Therefore, if set --max-jobs=$MAX_JOBS and don't set --cores, Guix could theoretically spin up $MAX_JOBS * $(nproc) number of threads, and that's no good. So we could either default to --cores=1, --max-jobs=$MAX_JOBS - Pro: --cores=1 means that `make` will be invoked with `-j1`, avoiding problems with package whose build systems and test suites break when running multi-threaded. - Con: There will be times when only 1 or 2 derivations can be built at a time, because the rest of the dependency graph all depend on those 1 or 2 derivations. During these times, the machine will be severely under-utilized. or --cores=$MAX_JOBS, --max-jobs=1 - Pro: We don't encounter prolonged periods of severe under-utilization mentioned above. - Con: Many packages' build systems and test suites break when running multi-threaded. or --cores=1, --max-jobs=1 and let the user override with $ADDITIONAL_GUIX_COMMON_FLAGS --- contrib/guix/README.md | 58 +++++++++++++++++++++++++++++++++++++++---- contrib/guix/guix-build.sh | 12 ++++----- contrib/guix/libexec/build.sh | 8 +++--- 3 files changed, 63 insertions(+), 15 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 1122ec9ba5..133f250ad2 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -80,6 +80,50 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` +### Controlling the number of threads used by `guix` build commands + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that there is a +`--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + ## Usage ### As a Tool for Deterministic Builds @@ -125,12 +169,16 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). -* _**MAX_JOBS**_ +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` - Override the maximum number of jobs to run simultaneously, you might want to - do so on a memory-limited machine. This may be passed to `make` as in `make - --jobs="$MAX_JOBS"` or `xargs` as in `xargs -P"$MAX_JOBS"`. _(defaults to the - value of `nproc` outside the container)_ + _(defaults to the value of `nproc` outside the container)_ * _**SOURCE_DATE_EPOCH**_ diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 7c32fb5726..430b7c3209 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -134,7 +134,7 @@ done # Determine the maximum number of jobs to run simultaneously (overridable by # environment) -MAX_JOBS="${MAX_JOBS:-$(nproc)}" +JOBS="${JOBS:-$(nproc)}" # Usage: host_to_commonname HOST # @@ -152,7 +152,7 @@ host_to_commonname() { # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do - make -C "${PWD}/depends" -j"$MAX_JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done # Determine the reference time used for determinism (overridable by environment) @@ -164,7 +164,7 @@ time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ @@ -218,7 +218,7 @@ for host in $HOSTS; do cat << EOF INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${MAX_JOBS:?not set} jobs + ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' @@ -304,12 +304,12 @@ EOF ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ ${SDK_PATH:+--share="$SDK_PATH"} \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ + JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 051066a6a0..0301d10522 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -26,7 +26,7 @@ cat << EOF Required environment variables as seen inside the container: HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - MAX_JOBS: ${MAX_JOBS:?not set} + JOBS: ${JOBS:?not set} DISTSRC: ${DISTSRC:?not set} OUTDIR: ${OUTDIR:?not set} EOF @@ -173,7 +173,7 @@ esac #################### # Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ +make -C depends --jobs="$JOBS" HOST="$HOST" \ ${V:+V=1} \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ @@ -267,7 +267,7 @@ mkdir -p "$DISTSRC" sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool # Build Bitcoin Core - make --jobs="$MAX_JOBS" ${V:+V=1} + make --jobs="$JOBS" ${V:+V=1} # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} @@ -344,7 +344,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac -- cgit v1.2.3 From 3e9982ab3877eb8fe0a8c0cb3d847ac0913c7336 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 26 Feb 2021 17:38:35 -0500 Subject: contrib: Silence git-describe when looking for tag Otherwise, it prints a rather disturbing message to stderr: fatal: no tag exactly matches '' --- contrib/gitian-descriptors/assign_DISTNAME | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME index a2ca768aaa..330fbc041b 100755 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -4,7 +4,7 @@ # # A helper script to be sourced into the gitian descriptors -if RECENT_TAG="$(git describe --exact-match HEAD)"; then +if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then VERSION="${RECENT_TAG#v}" else VERSION="$(git rev-parse --short=12 HEAD)" -- cgit v1.2.3 From e5b49a01f5d0f631e7f08f86ca8a2c2b8213319f Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 26 Feb 2021 17:39:44 -0500 Subject: guix: Create windeploy inside distsrc-* ./windeploy is a "working directory", and therefore belongs inside distsrc-*. Many people have noticed their Guix builds failing after hours simply because they did not remove windeploy (but did remove the distsrc-* directories). --- contrib/guix/libexec/build.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 0301d10522..9888a84c9d 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -394,21 +394,21 @@ mkdir -p "$DISTSRC" || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) ;; esac - ) -) + ) # $DISTSRC/installed -case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) - ) - ;; -esac + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC -- cgit v1.2.3 From 7753357a7bae98ec775c707b9dec4cea1e945802 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 12:36:55 -0400 Subject: guix: Add source-able bash prelude and utils --- contrib/guix/libexec/prelude.bash | 60 +++++++++++++++++++++++++++++++++ contrib/shell/git-utils.bash | 14 ++++++++ contrib/shell/realpath.bash | 71 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 contrib/guix/libexec/prelude.bash create mode 100644 contrib/shell/git-utils.bash create mode 100644 contrib/shell/realpath.bash (limited to 'contrib') diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..33a319012c --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=../../shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=../../shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invokable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Set common variables +################ + +VERSION="${VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..37bac1f38d --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} -- cgit v1.2.3 From 4eccf063b252bfe256cf72d363a24cf0183e926e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 14:33:10 -0400 Subject: guix: Remove guix-build.sh filename extension --- contrib/guix/guix-build | 323 +++++++++++++++++++++++++++++++++++++++++++++ contrib/guix/guix-build.sh | 323 --------------------------------------------- 2 files changed, 323 insertions(+), 323 deletions(-) create mode 100755 contrib/guix/guix-build delete mode 100755 contrib/guix/guix-build.sh (limited to 'contrib') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build new file mode 100755 index 0000000000..430b7c3209 --- /dev/null +++ b/contrib/guix/guix-build @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +################### +## Sanity Checks ## +################### + +################ +# Check 1: Make sure that we can invoke required tools +################ +for cmd in git make guix cat mkdir curl; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi +done + +################ +# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# Check 3: Make sure that we're not in a dirty worktree +################ +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +else + GIT_COMMIT=$(git rev-parse --short=12 HEAD) +fi + +################ +# Check 4: Make sure that build directories do not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin18}" + +DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + + mkdir -p "$DISTSRC_BASE" +fi + +################ +# Check 5: When building for darwin, make sure that the macOS SDK exists +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + exit 1 + fi + ;; + esac +done + +######### +# Setup # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" + +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://github.com/dongcarl/guix.git \ + --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ + --cores="$JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + +# Make sure an output directory exists for our builds +OUTDIR="${OUTDIR:-${PWD}/output}" +[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" + +######### +# Build # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + +# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we +# can map them into the container +[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH" +[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE" +[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH" + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '${OUTDIR:?not set}' + ...bind-mounted in container to: '/outdir' +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR"=/outdir \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR=/outdir \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh deleted file mode 100755 index 430b7c3209..0000000000 --- a/contrib/guix/guix-build.sh +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -################### -## Sanity Checks ## -################### - -################ -# Check 1: Make sure that we can invoke required tools -################ -for cmd in git make guix cat mkdir curl; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi -done - -################ -# Check 2: Make sure GUIX_BUILD_OPTIONS is empty -################ -# -# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that -# can perform builds. This seems like what we want instead of -# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually -# _appended_ to normal command-line options. Meaning that they will take -# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. -# -# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's -# existence here and direct users of this script to use our (more flexible) -# custom environment variables. -if [ -n "$GUIX_BUILD_OPTIONS" ]; then -cat << EOF -Error: Environment variable GUIX_BUILD_OPTIONS is not empty: - '$GUIX_BUILD_OPTIONS' - -Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset -GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options -across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a -specific guix command. - -See contrib/guix/README.md for more details. -EOF -exit 1 -fi - -################ -# Check 3: Make sure that we're not in a dirty worktree -################ -if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then -cat << EOF -ERR: The current git worktree is dirty, which may lead to broken builds. - - Aborting... - -Hint: To make your git worktree clean, You may want to: - 1. Commit your changes, - 2. Stash your changes, or - 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on - using a dirty worktree -EOF -exit 1 -else - GIT_COMMIT=$(git rev-parse --short=12 HEAD) -fi - -################ -# Check 4: Make sure that build directories do not exist -################ - -# Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86_64-w64-mingw32 - x86_64-apple-darwin18}" - -DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" - -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" -} - -# Accumulate a list of build directories that already exist... -hosts_distsrc_exists="" -for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then - hosts_distsrc_exists+=" ${host}" - fi -done - -if [ -n "$hosts_distsrc_exists" ]; then -# ...so that we can print them out nicely in an error message -cat << EOF -ERR: Build directories for this commit already exist for the following platform - triples you're attempting to build, probably because of previous builds. - Please remove, or otherwise deal with them prior to starting another build. - - Aborting... - -EOF -for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" -done -exit 1 -else - - mkdir -p "$DISTSRC_BASE" -fi - -################ -# Check 5: When building for darwin, make sure that the macOS SDK exists -################ - -for host in $HOSTS; do - case "$host" in - *darwin*) - OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" - if [ -e "$OSX_SDK" ]; then - echo "Found macOS SDK at '${OSX_SDK}', using..." - else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." - exit 1 - fi - ;; - esac -done - -######### -# Setup # -######### - -# Determine the maximum number of jobs to run simultaneously (overridable by -# environment) -JOBS="${JOBS:-$(nproc)}" - -# Usage: host_to_commonname HOST -# -# HOST: The current platform triple we're building for -# -host_to_commonname() { - case "$1" in - *darwin*) echo osx ;; - *mingw*) echo win ;; - *linux*) echo linux ;; - *) exit 1 ;; - esac -} - -# Download the depends sources now as we won't have internet access in the build -# container -for host in $HOSTS; do - make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} -done - -# Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" - -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --cores="$JOBS" \ - --keep-failed \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - -# Make sure an output directory exists for our builds -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" - -######### -# Build # -######### - -# Function to be called when building for host ${1} and the user interrupts the -# build -int_trap() { -cat << EOF -** INT received while building ${1}, you may want to clean up the relevant - output, deploy, and distsrc-* directories before rebuilding - -Hint: To blow everything away, you may want to use: - - $ git clean -xdff --exclude='/depends/SDKs/*' - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory. Practically speaking, this means that all ignored -and untracked files and directories will be wiped, allowing you to start anew. -EOF -} - -# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we -# can map them into the container -[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH" -[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE" -[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH" - -# Deterministically build Bitcoin Core -# shellcheck disable=SC2153 -for host in $HOSTS; do - - # Display proper warning when the user interrupts the build - trap 'int_trap ${host}' INT - - ( - # Required for 'contrib/guix/manifest.scm' to output the right manifest - # for the particular $HOST we're building for - export HOST="$host" - - # shellcheck disable=SC2030 -cat << EOF -INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: - ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${JOBS:?not set} jobs - ...from worktree directory: '${PWD}' - ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '${OUTDIR:?not set}' - ...bind-mounted in container to: '/outdir' -EOF - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm'. - # - # Explanation of `guix environment` flags: - # - # --container run command within an isolated container - # - # Running in an isolated container minimizes build-time differences - # between machines and improves reproducibility - # - # --pure unset existing environment variables - # - # Same rationale as --container - # - # --no-cwd do not share current working directory with an - # isolated container - # - # When --container is specified, the default behavior is to share - # the current working directory with the isolated container at the - # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to - # '/home/satoshi/bitcoin/'). This means that the $PWD inside the - # container becomes a source of irreproducibility. --no-cwd disables - # this behaviour. - # - # --share=SPEC for containers, share writable host file system - # according to SPEC - # - # --share="$PWD"=/bitcoin - # - # maps our current working directory to /bitcoin - # inside the isolated container, which we later cd - # into. - # - # While we don't want to map our current working directory to the - # same exact path (as this introduces irreproducibility), we do want - # it to be at a _fixed_ path _somewhere_ inside the isolated - # container so that we have something to build. '/bitcoin' was - # chosen arbitrarily. - # - # ${SOURCES_PATH:+--share="$SOURCES_PATH"} - # - # make the downloaded depends sources path available - # inside the isolated container - # - # The isolated container has no network access as it's in a - # different network namespace from the main machine, so we have to - # make the downloaded depends sources available to it. The sources - # should have been downloaded prior to this invocation. - # - # --keep-failed keep build tree of failed builds - # - # When builds of the Guix environment itself (not Bitcoin Core) - # fail, it is useful for the build tree to be kept for debugging - # purposes. - # - # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} - # - # fetch substitute from SUBSTITUTE_URLS if they are - # authorized - # - # Depending on the user's security model, it may be desirable to use - # substitutes (pre-built packages) from servers that the user trusts. - # Please read the README.md in the same directory as this file for - # more information. - # - # shellcheck disable=SC2086,SC2031 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR"=/outdir \ - --expose="$(git rev-parse --git-common-dir)" \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${BASE_CACHE:+--share="$BASE_CACHE"} \ - ${SDK_PATH:+--share="$SDK_PATH"} \ - --cores="$JOBS" \ - --keep-failed \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - JOBS="$JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ - ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR=/outdir \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" - ) - -done -- cgit v1.2.3 From 7f401c953f8bb3574cec48561e13ef3b47dedc6e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 12:59:18 -0400 Subject: guix: Adapt guix-build to prelude, restructure hier --- contrib/guix/guix-build | 67 ++++++++++++++++++++++++++----------------- contrib/guix/libexec/build.sh | 9 +++--- 2 files changed, 44 insertions(+), 32 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 430b7c3209..a1c120f412 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -2,22 +2,26 @@ export LC_ALL=C set -e -o pipefail +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + ################### -## Sanity Checks ## +## SANITY CHECKS ## ################### ################ -# Check 1: Make sure that we can invoke required tools +# Required non-builtin commands should be invokable ################ -for cmd in git make guix cat mkdir curl; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi -done + +check_tools cat mkdir make git guix ################ -# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +# GUIX_BUILD_OPTIONS should be empty ################ # # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that @@ -45,8 +49,9 @@ exit 1 fi ################ -# Check 3: Make sure that we're not in a dirty worktree +# The git worktree should not be dirty ################ + if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then cat << EOF ERR: The current git worktree is dirty, which may lead to broken builds. @@ -60,12 +65,12 @@ Hint: To make your git worktree clean, You may want to: using a dirty worktree EOF exit 1 -else - GIT_COMMIT=$(git rev-parse --short=12 HEAD) fi +mkdir -p "$VERSION_BASE" + ################ -# Check 4: Make sure that build directories do not exist +# Build directories should not exist ################ # Default to building for all supported HOSTs (overridable by environment) @@ -73,14 +78,12 @@ export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu ri x86_64-w64-mingw32 x86_64-apple-darwin18}" -DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" - # Usage: distsrc_for_host HOST # # HOST: The current platform triple we're building for # distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" } # Accumulate a list of build directories that already exist... @@ -106,12 +109,11 @@ for host in $hosts_distsrc_exists; do done exit 1 else - mkdir -p "$DISTSRC_BASE" fi ################ -# Check 5: When building for darwin, make sure that the macOS SDK exists +# When building for darwin, the macOS SDK should exists ################ for host in $HOSTS; do @@ -129,7 +131,7 @@ for host in $HOSTS; do done ######### -# Setup # +# SETUP # ######### # Determine the maximum number of jobs to run simultaneously (overridable by @@ -172,11 +174,20 @@ time-machine() { } # Make sure an output directory exists for our builds -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: outdir_for_host HOST +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}" +} + ######### -# Build # +# BUILD # ######### # Function to be called when building for host ${1} and the user interrupts the @@ -216,15 +227,15 @@ for host in $HOSTS; do # shellcheck disable=SC2030 cat << EOF -INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '${OUTDIR:?not set}' - ...bind-mounted in container to: '/outdir' + ...outdirting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build @@ -299,7 +310,7 @@ EOF --no-cwd \ --share="$PWD"=/bitcoin \ --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR"=/outdir \ + --share="$OUTDIR_BASE"=/outdir-base \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ @@ -309,6 +320,7 @@ EOF ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ @@ -316,7 +328,8 @@ EOF ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR=/outdir \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 9888a84c9d..bb83516f89 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -24,6 +24,8 @@ fi # Check that required environment variables are set cat << EOF Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} JOBS: ${JOBS:?not set} @@ -198,11 +200,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ # Source Tarball Building # ########################### -# Define DISTNAME variable. -# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME -source contrib/gitian-descriptors/assign_DISTNAME - -GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then @@ -275,6 +273,7 @@ mkdir -p "$DISTSRC" # version symbols for Linux distro back-compatibility. make -C src --jobs=1 check-symbols ${V:+V=1} + mkdir -p ${OUTDIR} # Make the os-specific installers case "$HOST" in *mingw*) -- cgit v1.2.3 From d55a1056ee565afed64e42d6f6efb6b0adc5599b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:01:07 -0400 Subject: guix: Add troubleshooting documentation entries --- contrib/guix/README.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 133f250ad2..2d128c7ed6 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -265,6 +265,57 @@ To use dongcarl's substitute server for Bitcoin Core builds after having export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` +## Troubleshooting + +### Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion: + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild stil did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +#### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + ## FAQ ### How can I trust the binary installation? -- cgit v1.2.3 From 39741128d3775d198dbee34dc827353bfd18acd8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:03:46 -0400 Subject: guix: Supply --link-profile --- contrib/guix/guix-build | 1 + 1 file changed, 1 insertion(+) (limited to 'contrib') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index a1c120f412..6c0930df18 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -317,6 +317,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ -- cgit v1.2.3 From c1ae726a13ecfa5e7e9fdc3030a8110b8bb263f8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:07:40 -0400 Subject: guix: More thoroughly control native toolchain --- contrib/guix/libexec/build.sh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bb83516f89..7d4609c17a 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -54,16 +54,34 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" -export CPATH="${NATIVE_GCC}/include" + +unset LIBRARY_PATH +unset CPATH unset C_INCLUDE_PATH unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + case "$HOST" in *darwin*) # When targeting darwin, zlib is required by native_libdmg-hfsplus. zlib_store_path=$(store_path "zlib") - export LIBRARY_PATH="${zlib_store_path}/lib:${LIBRARY_PATH}" - export CPATH="${zlib_store_path}/include:${CPATH}" + + prepend_to_search_env_var LIBRARY_PATH "${zlib_store_path}/lib" + prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" esac # Set environment variables to point the CROSS toolchain to the right -- cgit v1.2.3 From 1742f8e12d163852df09575e03edcd3db73198ee Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:00 -0400 Subject: guix: Add early health check for guix-daemon --- contrib/guix/guix-build | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 6c0930df18..7f74524ddc 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -130,6 +130,33 @@ for host in $HOSTS; do esac done +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + ######### # SETUP # ######### -- cgit v1.2.3 From 1aec0eda8fd31a57b0621eea616398017c2ead98 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:21 -0400 Subject: guix: Fallback to local build for substitute-enabled Guix users --- contrib/guix/guix-build | 2 ++ 1 file changed, 2 insertions(+) (limited to 'contrib') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 7f74524ddc..d817fb74ba 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -195,6 +195,7 @@ time-machine() { --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ --cores="$JOBS" \ --keep-failed \ + --fallback \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" @@ -344,6 +345,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --fallback \ --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- cgit v1.2.3 From ca85679eb43b8375a95d82101977829d08fb1e1b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:15:08 -0400 Subject: guix: Use clang-toolchain instead of clang --- contrib/guix/manifest.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 4228532cb1..3fc3f4d404 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -237,5 +237,5 @@ chain for " target " development.")) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) + (list clang-toolchain-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) (else '()))))) -- cgit v1.2.3 From 65176ab5730dff34466caaecdd292625ef8294fc Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 15:33:18 -0400 Subject: guix: Remove codesign_allocate+pagestuff from unsigned tarball --- contrib/guix/libexec/build.sh | 3 --- 1 file changed, 3 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 7d4609c17a..4a6b7792a9 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -323,9 +323,6 @@ mkdir -p "$DISTSRC" osx_volname \ contrib/macdeploy/detached-sig-{apply,create}.sh \ "${BASEPREFIX}/${HOST}"/native/bin/dmg - for util in codesign_allocate pagestuff; do - cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util" - done mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" -- cgit v1.2.3 From 7476b46f1893a4858616d2a8456a7c43238851ed Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 23 Mar 2021 12:59:59 -0400 Subject: guix: Build dmg as a static binary This relatively easy change eliminates all runtime dependencies (except for the kernel) for dmg, which is the only native build tool that gets put in our output tarballs. This allows much more flexibility when constructing the codesigning environment, and is much more robust. --- contrib/guix/libexec/build.sh | 6 ++++-- contrib/guix/manifest.scm | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'contrib') diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 4a6b7792a9..4239c3d475 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -54,6 +54,7 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" unset LIBRARY_PATH unset CPATH @@ -62,7 +63,7 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" @@ -76,8 +77,9 @@ case "$HOST" in *darwin*) # When targeting darwin, zlib is required by native_libdmg-hfsplus. zlib_store_path=$(store_path "zlib") + zlib_static_store_path=$(store_path "zlib" static) - prepend_to_search_env_var LIBRARY_PATH "${zlib_store_path}/lib" + prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 3fc3f4d404..910a9dd6f6 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -214,6 +214,7 @@ chain for " target " development.")) gzip xz zlib + (list zlib "static") ;; Build tools gnu-make libtool @@ -227,7 +228,8 @@ chain for " target " development.")) ;; Git git ;; Native gcc 7 toolchain - gcc-toolchain-7) + gcc-toolchain-7 + (list gcc-toolchain-7 "static")) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows -- cgit v1.2.3