diff options
| -rw-r--r-- | contrib/guix/README.md | 58 | ||||
| -rwxr-xr-x | contrib/guix/guix-build.sh | 12 | ||||
| -rw-r--r-- | contrib/guix/libexec/build.sh | 8 |
3 files changed, 63 insertions, 15 deletions
diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 1122ec9ba5..133f250ad2 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -80,6 +80,50 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` +### Controlling the number of threads used by `guix` build commands + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that there is a +`--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + ## Usage ### As a Tool for Deterministic Builds @@ -125,12 +169,16 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). -* _**MAX_JOBS**_ +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` - Override the maximum number of jobs to run simultaneously, you might want to - do so on a memory-limited machine. This may be passed to `make` as in `make - --jobs="$MAX_JOBS"` or `xargs` as in `xargs -P"$MAX_JOBS"`. _(defaults to the - value of `nproc` outside the container)_ + _(defaults to the value of `nproc` outside the container)_ * _**SOURCE_DATE_EPOCH**_ diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 7c32fb5726..430b7c3209 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -134,7 +134,7 @@ done # Determine the maximum number of jobs to run simultaneously (overridable by # environment) -MAX_JOBS="${MAX_JOBS:-$(nproc)}" +JOBS="${JOBS:-$(nproc)}" # Usage: host_to_commonname HOST # @@ -152,7 +152,7 @@ host_to_commonname() { # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do - make -C "${PWD}/depends" -j"$MAX_JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done # Determine the reference time used for determinism (overridable by environment) @@ -164,7 +164,7 @@ time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ @@ -218,7 +218,7 @@ for host in $HOSTS; do cat << EOF INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${MAX_JOBS:?not set} jobs + ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' @@ -304,12 +304,12 @@ EOF ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ ${SDK_PATH:+--share="$SDK_PATH"} \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ + JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 051066a6a0..0301d10522 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -26,7 +26,7 @@ cat << EOF Required environment variables as seen inside the container: HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - MAX_JOBS: ${MAX_JOBS:?not set} + JOBS: ${JOBS:?not set} DISTSRC: ${DISTSRC:?not set} OUTDIR: ${OUTDIR:?not set} EOF @@ -173,7 +173,7 @@ esac #################### # Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ +make -C depends --jobs="$JOBS" HOST="$HOST" \ ${V:+V=1} \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ @@ -267,7 +267,7 @@ mkdir -p "$DISTSRC" sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool # Build Bitcoin Core - make --jobs="$MAX_JOBS" ${V:+V=1} + make --jobs="$JOBS" ${V:+V=1} # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} @@ -344,7 +344,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac |