Merge #21375: guix: Misc feedback-based fixes + hier restructuring

7476b46f1893a4858616d2a8456a7c43238851ed guix: Build dmg as a static binary (Carl Dong)
06d6cf6784421290e6235fe8684d5e08ed6f1b62 depends: libdmg-hfsplus: Skip CMake RPATH patching (Carl Dong)
65176ab5730dff34466caaecdd292625ef8294fc guix: Remove codesign_allocate+pagestuff from unsigned tarball (Carl Dong)
ca85679eb43b8375a95d82101977829d08fb1e1b guix: Use clang-toolchain instead of clang (Carl Dong)
1aec0eda8fd31a57b0621eea616398017c2ead98 guix: Fallback to local build for substitute-enabled Guix users (Carl Dong)
1742f8e12d163852df09575e03edcd3db73198ee guix: Add early health check for guix-daemon (Carl Dong)
c1ae726a13ecfa5e7e9fdc3030a8110b8bb263f8 guix: More thoroughly control native toolchain (Carl Dong)
39741128d3775d198dbee34dc827353bfd18acd8 guix: Supply --link-profile (Carl Dong)
d55a1056ee565afed64e42d6f6efb6b0adc5599b guix: Add troubleshooting documentation entries (Carl Dong)
7f401c953f8bb3574cec48561e13ef3b47dedc6e guix: Adapt guix-build to prelude, restructure hier (Carl Dong)
4eccf063b252bfe256cf72d363a24cf0183e926e guix: Remove guix-build.sh filename extension (Carl Dong)
7753357a7bae98ec775c707b9dec4cea1e945802 guix: Add source-able bash prelude and utils (Carl Dong)
e5b49a01f5d0f631e7f08f86ca8a2c2b8213319f guix: Create windeploy inside distsrc-* (Carl Dong)
3e9982ab3877eb8fe0a8c0cb3d847ac0913c7336 contrib: Silence git-describe when looking for tag (Carl Dong)
d5a71e97853ea9e1b879e8c76bfb01d4bef33172 guix: Use --cores instead of --max-jobs (Carl Dong)

Pull request description:

  This PR addresses a few hiccups encountered by the brave souls who've been experimenting with the Guix scripts:
  - Resolves confusion between `--cores=` and `--max-jobs=`
    - `guix`'s `--cores=` actually corresponds to make's `--jobs=`, so let's just control `--cores=` with our overridable env var
  - `git-describe` will scream `fatal: no tag exactly matches '<hash>'` when looking for a tag, but we don't care, so silence that
  - `windeploy/unsigned` should be inside `distsrc-*` and created idempotently (sorry I know this one annoyed people)
  - Add troubleshooting documentation to `README.md`
  - Add early health check for `guix-daemon` in case user forgot to start a `guix-daemon`
  - Depending on configuration, a `--fallback` flag may be needed to tell Guix to not fail if substitutes fail but fallback to building locally
  - `codesign_allocate` and `pagestuff` are now unnecessary for codesigning as we're now using `signapple`

  A few robustness changes are also included:
  - We supply the `--link-profile` flag, as some Guix packages may expect the profile to be available under `$HOME/.guix-profile`
  - We now clear and manually set all toolchain-related env vars (e.g. `C*_INCLUDE_PATH`) ourselves, after patching a Qt::moc bug
  - We use the native `clang-toolchain` package for darwin builds instead of `clang`, lining up with all our other toolchain packages.

  Finally, we restructure the guix building hierarchy such that it looks something like:
  ```
  guix-build-<short-hash-or-version-tag>
  ├── distsrc-<short-hash-or-version-tag>-${HOST}
  │   ├── contrib
  │   ├── depends
  │   ├── src
  │   └── ...
  ├── distsrc-<short-hash-or-version-tag>-...
  └── output
      ├── dist-archive
      │   └── bitcoin-<short-hash-or-version-tag>.tar.gz
      ├── *-linux-*
      │   ├── bitcoin-<short-hash-or-version-tag>-*-linux-*-debug.tar.gz
      │   └── bitcoin-<short-hash-or-version-tag>-*-linux-*.tar.gz
      ├── x86_64-apple-darwin18
      │   ├── bitcoin-<short-hash-or-version-tag>-osx64.tar.gz
      │   ├── bitcoin-<short-hash-or-version-tag>-osx-unsigned.dmg
      │   └── bitcoin-<short-hash-or-version-tag>-osx-unsigned.tar.gz
      └── x86_64-w64-mingw32
          ├── bitcoin-<short-hash-or-version-tag>-win64-debug.zip
          ├── bitcoin-<short-hash-or-version-tag>-win64-setup-unsigned.exe
          ├── bitcoin-<short-hash-or-version-tag>-win64.zip
          └── bitcoin-<short-hash-or-version-tag>-win-unsigned.tar.gz
  ```
  Separating guix builds by their version identifier (basically namespacing them) allows us to change the layout in the future without worry about potential naming conflicts.

ACKs for top commit:
  sipa:
    ACK 7476b46f1893a4858616d2a8456a7c43238851ed
  laanwj:
    ACK 7476b46f1893a4858616d2a8456a7c43238851ed

Tree-SHA512: 0e899aa941aafdf552b2a7e8a08131ee9283180bbef7334439e2461a02aa7235ab7b9ca9c149b80fc5d0a9f4bbd35bc80fcee26197c0836ba8eaf2d86ffa0386

1 files changed, 49 insertions, 33 deletions

diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh
index 051066a6a0..4239c3d475 100644
--- a/contrib/guix/libexec/build.sh
+++ b/contrib/guix/libexec/build.sh
@@ -24,9 +24,11 @@ fi
 # Check that required environment variables are set
 cat << EOF
 Required environment variables as seen inside the container:
+    DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set}
+    DISTNAME: ${DISTNAME:?not set}
     HOST: ${HOST:?not set}
     SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set}
-    MAX_JOBS: ${MAX_JOBS:?not set}
+    JOBS: ${JOBS:?not set}
     DISTSRC: ${DISTSRC:?not set}
     OUTDIR: ${OUTDIR:?not set}
 EOF
@@ -52,16 +54,36 @@ store_path() {
 # Set environment variables to point the NATIVE toolchain to the right
 # includes/libs
 NATIVE_GCC="$(store_path gcc-toolchain)"
-export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64"
-export CPATH="${NATIVE_GCC}/include"
+NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)"
+
+unset LIBRARY_PATH
+unset CPATH
 unset C_INCLUDE_PATH
 unset CPLUS_INCLUDE_PATH
+unset OBJC_INCLUDE_PATH
+unset OBJCPLUS_INCLUDE_PATH
+
+export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64"
+export C_INCLUDE_PATH="${NATIVE_GCC}/include"
+export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include"
+export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include"
+export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include"
+
+prepend_to_search_env_var() {
+    export "${1}=${2}${!1:+:}${!1}"
+}
+
 case "$HOST" in
     *darwin*)
         # When targeting darwin, zlib is required by native_libdmg-hfsplus.
         zlib_store_path=$(store_path "zlib")
-        export LIBRARY_PATH="${zlib_store_path}/lib:${LIBRARY_PATH}"
-        export CPATH="${zlib_store_path}/include:${CPATH}"
+        zlib_static_store_path=$(store_path "zlib" static)
+
+        prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib"
+        prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include"
+        prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include"
+        prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include"
+        prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include"
 esac
 
 # Set environment variables to point the CROSS toolchain to the right
@@ -173,7 +195,7 @@ esac
 ####################
 
 # Build the depends tree, overriding variables that assume multilib gcc
-make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \
+make -C depends --jobs="$JOBS" HOST="$HOST" \
                                    ${V:+V=1} \
                                    ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \
                                    ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \
@@ -198,11 +220,7 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \
 # Source Tarball Building #
 ###########################
 
-# Define DISTNAME variable.
-# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
-source contrib/gitian-descriptors/assign_DISTNAME
-
-GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
+GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz"
 
 # Create the source tarball if not already there
 if [ ! -e "$GIT_ARCHIVE" ]; then
@@ -267,7 +285,7 @@ mkdir -p "$DISTSRC"
     sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool
 
     # Build Bitcoin Core
-    make --jobs="$MAX_JOBS" ${V:+V=1}
+    make --jobs="$JOBS" ${V:+V=1}
 
     # Perform basic ELF security checks on a series of executables.
     make -C src --jobs=1 check-security ${V:+V=1}
@@ -275,6 +293,7 @@ mkdir -p "$DISTSRC"
     # version symbols for Linux distro back-compatibility.
     make -C src --jobs=1 check-symbols  ${V:+V=1}
 
+    mkdir -p ${OUTDIR}
     # Make the os-specific installers
     case "$HOST" in
         *mingw*)
@@ -306,9 +325,6 @@ mkdir -p "$DISTSRC"
                 osx_volname \
                 contrib/macdeploy/detached-sig-{apply,create}.sh \
                 "${BASEPREFIX}/${HOST}"/native/bin/dmg
-            for util in codesign_allocate pagestuff; do
-                cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util"
-            done
             mv --target-directory="unsigned-app-${HOST}" dist
             (
                 cd "unsigned-app-${HOST}"
@@ -344,7 +360,7 @@ mkdir -p "$DISTSRC"
                 {
                     find "${DISTNAME}/bin" -type f -executable -print0
                     find "${DISTNAME}/lib" -type f -print0
-                } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg
+                } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg
                 ;;
         esac
 
@@ -394,21 +410,21 @@ mkdir -p "$DISTSRC"
                     || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 )
                 ;;
         esac
-    )
-)
+    )  # $DISTSRC/installed
 
-case "$HOST" in
-    *mingw*)
-        cp -rf --target-directory=. contrib/windeploy
-        (
-            cd ./windeploy
-            mkdir unsigned
-            cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe"
-            find . -print0 \
-                | sort --zero-terminated \
-                | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
-                | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \
-                || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 )
-        )
-        ;;
-esac
+    case "$HOST" in
+        *mingw*)
+            cp -rf --target-directory=. contrib/windeploy
+            (
+                cd ./windeploy
+                mkdir -p unsigned
+                cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe"
+                find . -print0 \
+                    | sort --zero-terminated \
+                    | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                    | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 )
+            )
+            ;;
+    esac
+)  # $DISTSRC