From 4eccf063b252bfe256cf72d363a24cf0183e926e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 14:33:10 -0400 Subject: guix: Remove guix-build.sh filename extension --- contrib/guix/guix-build | 323 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100755 contrib/guix/guix-build (limited to 'contrib/guix/guix-build') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build new file mode 100755 index 0000000000..430b7c3209 --- /dev/null +++ b/contrib/guix/guix-build @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +################### +## Sanity Checks ## +################### + +################ +# Check 1: Make sure that we can invoke required tools +################ +for cmd in git make guix cat mkdir curl; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi +done + +################ +# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# Check 3: Make sure that we're not in a dirty worktree +################ +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +else + GIT_COMMIT=$(git rev-parse --short=12 HEAD) +fi + +################ +# Check 4: Make sure that build directories do not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin18}" + +DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + + mkdir -p "$DISTSRC_BASE" +fi + +################ +# Check 5: When building for darwin, make sure that the macOS SDK exists +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + exit 1 + fi + ;; + esac +done + +######### +# Setup # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" + +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://github.com/dongcarl/guix.git \ + --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ + --cores="$JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + +# Make sure an output directory exists for our builds +OUTDIR="${OUTDIR:-${PWD}/output}" +[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" + +######### +# Build # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + +# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we +# can map them into the container +[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH" +[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE" +[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH" + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '${OUTDIR:?not set}' + ...bind-mounted in container to: '/outdir' +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR"=/outdir \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR=/outdir \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) + +done -- cgit v1.2.3 From 7f401c953f8bb3574cec48561e13ef3b47dedc6e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 12:59:18 -0400 Subject: guix: Adapt guix-build to prelude, restructure hier --- contrib/guix/guix-build | 67 +++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 27 deletions(-) (limited to 'contrib/guix/guix-build') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 430b7c3209..a1c120f412 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -2,22 +2,26 @@ export LC_ALL=C set -e -o pipefail +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + ################### -## Sanity Checks ## +## SANITY CHECKS ## ################### ################ -# Check 1: Make sure that we can invoke required tools +# Required non-builtin commands should be invokable ################ -for cmd in git make guix cat mkdir curl; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi -done + +check_tools cat mkdir make git guix ################ -# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +# GUIX_BUILD_OPTIONS should be empty ################ # # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that @@ -45,8 +49,9 @@ exit 1 fi ################ -# Check 3: Make sure that we're not in a dirty worktree +# The git worktree should not be dirty ################ + if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then cat << EOF ERR: The current git worktree is dirty, which may lead to broken builds. @@ -60,12 +65,12 @@ Hint: To make your git worktree clean, You may want to: using a dirty worktree EOF exit 1 -else - GIT_COMMIT=$(git rev-parse --short=12 HEAD) fi +mkdir -p "$VERSION_BASE" + ################ -# Check 4: Make sure that build directories do not exist +# Build directories should not exist ################ # Default to building for all supported HOSTs (overridable by environment) @@ -73,14 +78,12 @@ export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu ri x86_64-w64-mingw32 x86_64-apple-darwin18}" -DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" - # Usage: distsrc_for_host HOST # # HOST: The current platform triple we're building for # distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" } # Accumulate a list of build directories that already exist... @@ -106,12 +109,11 @@ for host in $hosts_distsrc_exists; do done exit 1 else - mkdir -p "$DISTSRC_BASE" fi ################ -# Check 5: When building for darwin, make sure that the macOS SDK exists +# When building for darwin, the macOS SDK should exists ################ for host in $HOSTS; do @@ -129,7 +131,7 @@ for host in $HOSTS; do done ######### -# Setup # +# SETUP # ######### # Determine the maximum number of jobs to run simultaneously (overridable by @@ -172,11 +174,20 @@ time-machine() { } # Make sure an output directory exists for our builds -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: outdir_for_host HOST +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}" +} + ######### -# Build # +# BUILD # ######### # Function to be called when building for host ${1} and the user interrupts the @@ -216,15 +227,15 @@ for host in $HOSTS; do # shellcheck disable=SC2030 cat << EOF -INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '${OUTDIR:?not set}' - ...bind-mounted in container to: '/outdir' + ...outdirting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build @@ -299,7 +310,7 @@ EOF --no-cwd \ --share="$PWD"=/bitcoin \ --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR"=/outdir \ + --share="$OUTDIR_BASE"=/outdir-base \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ @@ -309,6 +320,7 @@ EOF ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ @@ -316,7 +328,8 @@ EOF ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR=/outdir \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) -- cgit v1.2.3 From 39741128d3775d198dbee34dc827353bfd18acd8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:03:46 -0400 Subject: guix: Supply --link-profile --- contrib/guix/guix-build | 1 + 1 file changed, 1 insertion(+) (limited to 'contrib/guix/guix-build') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index a1c120f412..6c0930df18 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -317,6 +317,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ -- cgit v1.2.3 From 1742f8e12d163852df09575e03edcd3db73198ee Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:00 -0400 Subject: guix: Add early health check for guix-daemon --- contrib/guix/guix-build | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'contrib/guix/guix-build') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 6c0930df18..7f74524ddc 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -130,6 +130,33 @@ for host in $HOSTS; do esac done +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + ######### # SETUP # ######### -- cgit v1.2.3 From 1aec0eda8fd31a57b0621eea616398017c2ead98 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:21 -0400 Subject: guix: Fallback to local build for substitute-enabled Guix users --- contrib/guix/guix-build | 2 ++ 1 file changed, 2 insertions(+) (limited to 'contrib/guix/guix-build') diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 7f74524ddc..d817fb74ba 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -195,6 +195,7 @@ time-machine() { --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ --cores="$JOBS" \ --keep-failed \ + --fallback \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" @@ -344,6 +345,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --fallback \ --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- cgit v1.2.3