diff options
| -rw-r--r-- | Makefile.am | 2 | ||||
| -rwxr-xr-x | contrib/guix/guix-build | 100 | ||||
| -rwxr-xr-x | contrib/guix/guix-clean | 83 | ||||
| -rw-r--r-- | contrib/guix/libexec/prelude.bash | 6 | ||||
| -rw-r--r-- | depends/Makefile | 2 | ||||
| -rw-r--r-- | src/Makefile.am | 2 |
6 files changed, 175 insertions, 20 deletions
diff --git a/Makefile.am b/Makefile.am index be62c3e6a9..5e453b9ae1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4,7 +4,7 @@ # Pattern rule to print variables, e.g. make print-top_srcdir print-%: - @echo '$*' = '$($*)' + @echo '$*'='$($*)' ACLOCAL_AMFLAGS = -I build-aux/m4 SUBDIRS = src diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 7e436e4246..5b3c20b234 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -103,6 +103,14 @@ ERR: Build directories for this commit already exist for the following platform Aborting... +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. EOF for host in $hosts_distsrc_exists; do echo " ${host} '$(distsrc_for_host "$host")'" @@ -119,7 +127,7 @@ fi for host in $HOSTS; do case "$host" in *darwin*) - OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" if [ -e "$OSX_SDK" ]; then echo "Found macOS SDK at '${OSX_SDK}', using..." else @@ -178,12 +186,6 @@ host_to_commonname() { esac } -# Download the depends sources now as we won't have internet access in the build -# container -for host in $HOSTS; do - make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} -done - # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" @@ -201,10 +203,70 @@ time-machine() { -- "$@" } + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + make -C "${PWD}/depends" \ + --no-print-directory \ + -- $(printf "print-%s\n" $depends_precious_dir_names) + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + # Make sure an output directory exists for our builds OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" mkdir -p "$OUTDIR_BASE" +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + # Usage: outdir_for_host HOST # # HOST: The current platform triple we're building for @@ -213,6 +275,14 @@ outdir_for_host() { echo "${OUTDIR_BASE}/${1}" } +# Usage: profiledir_for_host HOST COMMAND +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${2}-${1}" +} + ######### # BUILD # @@ -223,24 +293,19 @@ outdir_for_host() { int_trap() { cat << EOF ** INT received while building ${1}, you may want to clean up the relevant - output, deploy, and distsrc-* directories before rebuilding + work directories (e.g. distsrc-*) before rebuilding Hint: To blow everything away, you may want to use: - $ git clean -xdff --exclude='/depends/SDKs/*' + $ ./contrib/guix/guix-clean Specifically, this will remove all files without an entry in the index, -excluding the SDK directory. Practically speaking, this means that all ignored -and untracked files and directories will be wiped, allowing you to start anew. +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. EOF } -# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we -# can map them into the container -[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH" -[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE" -[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH" - # Deterministically build Bitcoin Core # shellcheck disable=SC2153 for host in $HOSTS; do @@ -347,6 +412,7 @@ EOF --keep-failed \ --fallback \ --link-profile \ + --root="$(profiledir_for_host "${HOST}" build)" \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean new file mode 100755 index 0000000000..9fa17191e8 --- /dev/null +++ b/contrib/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##${1}}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 070bceff0e..9705607119 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -58,3 +58,9 @@ VERSION_BASE="${version_base_prefix}${VERSION}" # TOP DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/depends/Makefile b/depends/Makefile index dadb21515a..c5dfd1e8a7 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -2,7 +2,7 @@ # Pattern rule to print variables, e.g. make print-top_srcdir print-%: - @echo '$*' = '$($*)' + @echo '$*'='$($*)' # When invoking a sub-make, keep only the command line variable definitions # matching the pattern in the filter function. diff --git a/src/Makefile.am b/src/Makefile.am index 4e09c86ebd..b47bdfaa26 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,7 @@ # Pattern rule to print variables, e.g. make print-top_srcdir print-%: - @echo '$*' = '$($*)' + @echo '$*'='$($*)' DIST_SUBDIRS = secp256k1 univalue |