Merge bitcoin/bitcoin#22182: guix: Overhaul how guix-{attest,verify} works and hierarchy

e2c40a4ed5272d72fea997bd936fba28bb753226 guix-attest: Error out if SHA256SUMS is unexpected (Carl Dong)
4cc35daed557f38b080360a89036b2e97a6f78c2 Rewrite guix-{attest,verify} for new hier (Carl Dong)
28a9c9b83924f585b397f0f3b8e9e73780ac0ad6 Make SHA256SUMS fragment right after build (Carl Dong)

Pull request description:

  Based on:  #22075
  Code reviewers: I recommend reading the new `guix-{attest,verify}` files instead of trying to read the diff

  The following changes resolve many usability improvements which were pointed out to me:
  1. Some maintainers like to extract their "uncodesigned tarball" inside the `output/` directory, resulting in the older `guix-attest` mistakenly attesting to the extracted contents
  2. Maintainers whose GPG keys reside on an external smartcard often need to physically interact with the smartcard as a way to approve the signing operation, having one signature per platform means a lot of fidgeting
  3. Maintainers wishing to sign on a separate machine now has the option of transferring only a subtree of `output/`, namely `output/*/SHA256SUMS.part`, in order to perform a signature (you may need to specify an `$OUTDIR_BASE` env var)
  4. An `all.SHA256SUMS` file should be usable as the base `SHA256SUMS` in bitcoin core torrents and on the release server.

  For those who sign on an separate machine than the one you do builds on, the following steps will work:
  1. `env GUIX_SIGS_REPO=/home/achow101/guix.sigs SIGNER=achow101 NO_SIGN=1 ./contrib/guix/guix-attest`
  2. Copy `/home/achow101/guix.sigs/<tag>/achow101` (which does not yet have signatures) to signing machine
  3. Sign the `SHA256SUMS` files:
      ```bash
      for i in "<path-to-achow101>/*.SHA256SUMS"; do
          gpg --detach-sign --local-user "<your-key-here>" --armor --output "$i"{.asc,}
      done
      ```
  5. Upload `<path-to-achow101>` (now with signatures) to `guix.sigs`

  -----

  After this change, output directories will now include a `SHA256SUMS.part` fragment, created immediately after a successful build:
  ```
  output
  └── x86_64-w64-mingw32
      ├── bitcoin-4e069f7589da-win64-debug.zip
      ├── bitcoin-4e069f7589da-win64-setup-unsigned.exe
      ├── bitcoin-4e069f7589da-win64.zip
      ├── bitcoin-4e069f7589da-win-unsigned.tar.gz
      └── SHA256SUMS.part
  ```

  These `SHA256SUMS.part` fragments look something like:
  ```
  3ebd7262b1a0a5bb757fef1f70e7e14033c70f98c059bc4dbfee5d1992b25825  dist-archive/bitcoin-4e069f7589da.tar.gz
  def2e7d3de5ab3e3f955344e75151df4f33713f9101f5295bd13c9375bdf633b  x86_64-w64-mingw32/bitcoin-4e069f7589da-win64-debug.zip
  643049fe3ee4a4e83a1739607e67b11b7c9b1a66208a6f35a9ff634ba795500e  x86_64-w64-mingw32/bitcoin-4e069f7589da-win64-setup-unsigned.exe
  a247a1ccec0ccc2e138c648284bd01f6a761f2d8d6d07d91b5b4a6670ec3f288  x86_64-w64-mingw32/bitcoin-4e069f7589da-win-unsigned.tar.gz
  fab76a836dcc592e39c04fd2396696633fb6eb56e39ecbf6c909bd173ed4280c  x86_64-w64-mingw32/bitcoin-4e069f7589da-win64.zip
  ```

  Meaning that they are valid `SHA256SUMS` files when `sha256sum --check`'d at the `guix-build-*/output` directory level

  When `guix-attest` is invoked, these `SHA256SUMS.part` files are combined and sorted (by `-k2`, `LC_ALL=C`) to create:

  1. `noncodesigned.SHA256SUMS` for a manifest of all non-codesigned outputs, and
  3. `all.SHA256SUMS` for a manifest of all outputs including non-codesigned outputs

  Then both files are signed, resulting in the following `guix.sigs` hierarchy:
  ```
  4e069f7589da/
  └── dongcarl
      ├── all.SHA256SUMS
      ├── all.SHA256SUMS.asc
      ├── noncodesigned.SHA256SUMS
      └── noncodesigned.SHA256SUMS.asc
  ```

ACKs for top commit:
  achow101:
    ACK e2c40a4ed5272d72fea997bd936fba28bb753226
  hebasto:
    ACK e2c40a4ed5272d72fea997bd936fba28bb753226, tested on Linux Mint 20.1 (x86_64) with and w/o `NO_SIGN=1`. Changes in `contrib/guix/libexec/codesign.sh` and `contrib/guix/guix-verify` are reviewed only.

Tree-SHA512: 618aacefb0eb6595735a9ab6a98ea6598fce65f9ccf33fa1e7ef93bf140c0f6cfc16e34870c6aa3e4777dd3f004b92a82a994141879870141742df948ec59c1f

5 files changed, 193 insertions, 125 deletions

diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest
index 081d1c0465..c8cf73d400 100755
--- a/contrib/guix/guix-attest
+++ b/contrib/guix/guix-attest
@@ -99,24 +99,34 @@ fi
 # We should be able to find at least one output
 ################
 
-echo "Looking for build output directories in ${OUTDIR_BASE}"
+echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}"
 
 shopt -s nullglob
-OUTDIRS=( "${OUTDIR_BASE}"/* ) # This expands to an array of directories...
+sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories...
 shopt -u nullglob
 
-if (( ${#OUTDIRS[@]} )); then
-    echo "Found build output directories:"
-    for outdir in "${OUTDIRS[@]}"; do
+noncodesigned_fragments=()
+codesigned_fragments=()
+
+if (( ${#sha256sum_fragments[@]} )); then
+    echo "Found build output SHA256SUMS fragments:"
+    for outdir in "${sha256sum_fragments[@]}"; do
         echo "    '$outdir'"
+        case "$outdir" in
+            "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part)
+                codesigned_fragments+=("$outdir")
+                ;;
+            *)
+                noncodesigned_fragments+=("$outdir")
+                ;;
+        esac
     done
     echo
 else
-    echo "ERR: Could not find any build output directories in ${OUTDIR_BASE}"
+    echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}"
     exit 1
 fi
 
-
 ##############
 ##  Attest  ##
 ##############
@@ -126,82 +136,105 @@ fi
 #   HOST: The output directory being attested
 #
 out_name() {
-    basename "$1"
+    basename "$(dirname "$1")"
 }
 
-# Usage: out_sig_dir $outdir
-#
-#   outdir: The output directory being attested
-#
-out_sig_dir() {
-    echo "$GUIX_SIGS_REPO/$VERSION/$(out_name "$1")/$signer_name"
-}
+shasum_already_exists() {
+cat <<EOF
+--
 
-# Accumulate a list of signature directories that already exist...
-outdirs_already_attested_to=()
+ERR: An ${1} file already exists for '${VERSION}' and attests
+     differently. You likely previously attested to a partial build (e.g. one
+     where you specified the HOST environment variable).
 
-echo "Attesting to build outputs for version: '${VERSION}'"
-echo ""
+     See the diff above for more context.
 
-# MAIN LOGIC: Loop through each output for VERSION and attest to output in
-#             GUIX_SIGS_REPO as SIGNER, if attestation does not exist
-for outdir in "${OUTDIRS[@]}"; do
-    if [ -e "${outdir}/SKIPATTEST.TAG" ]; then
-        echo "${outname}: SKIPPING: Output directory marked with SKIPATTEST.TAG file"
-        continue
-    fi
-    outname="$(out_name "$outdir")"
-    outsigdir="$(out_sig_dir "$outdir")"
-    if [ -e "$outsigdir" ]; then
-        echo "${outname}: SKIPPING: Signature directory already exists in the specified guix.sigs repository"
-        outdirs_already_attested_to+=("$outdir")
-    else
-        # Clean up incomplete sigdir if something fails (likely gpg)
-        trap 'rm -rf "$outsigdir"' ERR
+Hint: You may wish to remove the existing attestations and their signatures by
+      invoking:
 
-        mkdir -p "$outsigdir"
+          rm '${PWD}/${1}'{,.asc}
 
-        (
-            cd "$outdir"
+      Then try running this script again.
 
-            if [ -e inputs.SHA256SUMS ]; then
-                echo "${outname}: Including existent input SHA256SUMS"
-                cat inputs.SHA256SUMS >> "$outsigdir"/SHA256SUMS
-            fi
+EOF
+}
 
-            echo "${outname}: Hashing build outputs to produce SHA256SUMS"
-            files="$(find -L . -type f ! -iname '*.SHA256SUMS')"
-            if [ -n "$files" ]; then
-                cut -c3- <<< "$files" | env LC_ALL=C sort | xargs sha256sum >> "$outsigdir"/SHA256SUMS
+echo "Attesting to build outputs for version: '${VERSION}'"
+echo ""
+
+outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name"
+mkdir -p "$outsigdir"
+(
+    cd "$outsigdir"
+
+    temp_noncodesigned="$(mktemp)"
+    trap 'rm -rf -- "$temp_noncodesigned"' EXIT
+
+    if (( ${#noncodesigned_fragments[@]} )); then
+        cat "${noncodesigned_fragments[@]}" \
+            | sort -u \
+            | sort -k2 \
+                   > "$temp_noncodesigned"
+        if [ -e noncodesigned.SHA256SUMS ]; then
+            # The SHA256SUMS already exists, make sure it's exactly what we
+            # expect, error out if not
+            if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then
+                echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date."
             else
-                echo "ERR: ${outname}: No outputs found in '${outdir}'"
+                shasum_already_exists noncodesigned.SHA256SUMS
                 exit 1
             fi
-        )
-        if [ -z "$NO_SIGN" ]; then
-            echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc"
-            gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS
         else
-            echo "${outname}: Not signing SHA256SUMS as \$NO_SIGN is not empty"
+            mv "$temp_noncodesigned" noncodesigned.SHA256SUMS
         fi
-        echo ""
-
-        trap - ERR  # Reset ERR trap
+    else
+        echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..."
+        exit 1
     fi
-done
-
-if (( ${#outdirs_already_attested_to[@]} )); then
-# ...so that we can print them out nicely in a warning message
-cat << EOF
 
-WARN: Signature directories from '$signer_name' already exist in the specified
-      guix.sigs repository for the following output directories and were
-      skipped:
+    temp_codesigned="$(mktemp)"
+    trap 'rm -rf -- "$temp_codesigned"' EXIT
+
+    if (( ${#codesigned_fragments[@]} )); then
+        # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is
+        #       not needed if there are no $codesigned_fragments
+        cat "${sha256sum_fragments[@]}" \
+            | sort -u \
+            | sort -k2 \
+                   > "$temp_codesigned"
+        if [ -e codesigned.SHA256SUMS ]; then
+            # The SHA256SUMS already exists, make sure it's exactly what we
+            # expect, error out if not
+            if diff -u all.SHA256SUMS "$temp_codesigned"; then
+                echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date."
+            else
+                shasum_already_exists all.SHA256SUMS
+                exit 1
+            fi
+        else
+            mv "$temp_codesigned" codesigned.SHA256SUMS
+        fi
+    else
+        # It is fine to have the codesigned outputs be missing (perhaps the
+        # detached codesigs have not been published yet), just print a log
+        # message instead of erroring out
+        echo "INFO: No codesigned outputs found for '${VERSION}', skipping..."
+    fi
 
-EOF
-for outdir in "${outdirs_already_attested_to[@]}"; do
-    echo "    '${outdir}'"
-    echo "    Corresponds to: '$(out_sig_dir "$outdir")'"
+    if [ -z "$NO_SIGN" ]; then
+        echo "Signing SHA256SUMS to produce SHA256SUMS.asc"
+        for i in *.SHA256SUMS; do
+            if [ ! -e "$i".asc ]; then
+                gpg --detach-sign \
+                    --local-user "$gpg_key_name" \
+                    --armor \
+                    --output "$i".asc "$i"
+            else
+                echo "Signature already there"
+            fi
+        done
+    else
+        echo "Not signing SHA256SUMS as \$NO_SIGN is not empty"
+    fi
     echo ""
-done
-fi
+)
diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify
index 629050956c..a6e2c4065e 100755
--- a/contrib/guix/guix-verify
+++ b/contrib/guix/guix-verify
@@ -56,58 +56,87 @@ cmd_usage
 exit 1
 fi
 
-################
-# We should be able to find at least one output
-################
+##############
+##  Verify  ##
+##############
 
 OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}"
-echo "Looking for output signature directories in '${OUTSIGDIR_BASE}'"
+echo "Looking for signature directories in '${OUTSIGDIR_BASE}'"
+echo ""
+
+# Usage: verify compare_manifest current_manifest
+verify() {
+    local compare_manifest="$1"
+    local current_manifest="$2"
+    if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then
+        echo "ERR: Failed to verify GPG signature in '${current_manifest}'"
+        echo ""
+        echo "Hint: Either the signature is invalid or the public key is missing"
+        echo ""
+    elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then
+        echo "ERR: The SHA256SUMS attestation in these two directories differ:"
+        echo "    '${compare_manifest}'"
+        echo "    '${current_manifest}'"
+        echo ""
+    else
+        echo "Verified: '${current_manifest}'"
+        echo ""
+    fi
+}
 
 shopt -s nullglob
-OUTSIGDIRS=( "$OUTSIGDIR_BASE"/* ) # This expands to an array of directories...
+all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS )
 shopt -u nullglob
 
-if (( ${#OUTSIGDIRS[@]} )); then
-    echo "Found output signature directories:"
-    for outsigdir in "${OUTSIGDIRS[@]}"; do
-        echo "    '$outsigdir'"
+echo "--------------------"
+echo ""
+if (( ${#all_noncodesigned[@]} )); then
+    compare_noncodesigned="${all_noncodesigned[0]}"
+
+    for current_manifest in "${all_noncodesigned[@]}"; do
+        verify "$compare_noncodesigned" "$current_manifest"
     done
-    echo
+
+    echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS"
+    echo ""
 else
-    echo "ERR: Could not find any output signature directories in ${OUTSIGDIR_BASE}"
-    exit 1
+    echo "WARN: No signature directories with noncodesigned.SHA256SUMS found"
+    echo ""
 fi
 
+shopt -s nullglob
+all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS )
+shopt -u nullglob
 
-##############
-##  Verify  ##
-##############
+echo "--------------------"
+echo ""
+if (( ${#all_all[@]} )); then
+    compare_all="${all_all[0]}"
 
-# MAIN LOGIC: Loop through each output for VERSION and check that the SHA256SUMS
-#             and SHA256SUMS.asc file match between signers, using the first
-#             available signer as the arbitrary comparison base.
-for outsigdir in "${OUTSIGDIRS[@]}"; do
-    echo "BEGIN: Checking output signatures for $(basename "$outsigdir")"
-    echo ""
-    signer_dirs=( "$outsigdir"/* ) # This expands to an array of directories...
-    compare_signer_dir="${signer_dirs[0]}" # ...we just want the first one
-    for current_signer_dir in "${signer_dirs[@]}"; do
-        if ! gpg --quiet --batch --verify "$current_signer_dir"/SHA256SUMS.asc "$current_signer_dir"/SHA256SUMS; then
-            echo "ERR: Failed to verify GPG signature in '${current_signer_dir}/SHA256SUMS.asc'"
-            echo ""
-            echo "Hint: Either the signature is invalid or the public key is missing"
-            echo ""
-        elif ! diff --report-identical "$compare_signer_dir"/SHA256SUMS "$current_signer_dir"/SHA256SUMS; then
-            echo "ERR: The SHA256SUMS attestation in these two directories differ:"
-            echo "    '${compare_signer_dir}'"
-            echo "    '${current_signer_dir}'"
-            echo ""
-        else
-            echo "Verified: '${current_signer_dir}'"
-            echo ""
-        fi
+    for current_manifest in "${all_all[@]}"; do
+        verify "$compare_all" "$current_manifest"
     done
-    echo "DONE: Checking output signatures for $(basename "$outsigdir")"
+
+    # Sanity check: there should be no entries that exist in
+    # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS
+    if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then
+        echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which"
+        echo "     do not exist in all.SHA256SUMS, something went very wrong."
+        exit 1
+    fi
+
+    echo "DONE: Checking output signatures for all.SHA256SUMS"
     echo ""
+else
+    echo "WARN: No signature directories with all.SHA256SUMS found"
+    echo ""
+fi
+
+echo "===================="
+echo ""
+if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then
+    echo "ERR: Unable to perform any verifications as no signature directories"
+    echo "     were found"
     echo ""
-done
+    exit 1
+fi
diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh
index 3073b41baf..6741328473 100755
--- a/contrib/guix/libexec/build.sh
+++ b/contrib/guix/libexec/build.sh
@@ -230,20 +230,7 @@ if [ ! -e "$GIT_ARCHIVE" ]; then
     git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD
 fi
 
-# tmpdir="$(mktemp -d)"
-# (
-#     cd "$tmpdir"
-#     mkdir -p inputs
-#     ln -sf --target-directory=inputs "$GIT_ARCHIVE"
-
-#     mkdir -p "$OUTDIR"
-#     find -L inputs -type f -print0 | xargs -0 sha256sum > "${OUTDIR}/inputs.SHA256SUMS"
-# )
-
 mkdir -p "$OUTDIR"
-cat << EOF > "$OUTDIR"/inputs.SHA256SUMS
-$(sha256sum "$GIT_ARCHIVE" | cut -d' ' -f1)  inputs/$(basename "$GIT_ARCHIVE")
-EOF
 
 ###########################
 # Binary Tarball Building #
@@ -450,3 +437,13 @@ mkdir -p "$DISTSRC"
 rm -rf "$ACTUAL_OUTDIR"
 mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \
     || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 )
+
+(
+    cd /outdir-base
+    {
+        echo "$GIT_ARCHIVE"
+        find "$ACTUAL_OUTDIR" -type f
+    } | xargs realpath --relative-base="$PWD" \
+      | xargs sha256sum \
+      | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part
+)
diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh
index 1822da7ca4..b1eec686ec 100755
--- a/contrib/guix/libexec/codesign.sh
+++ b/contrib/guix/libexec/codesign.sh
@@ -55,10 +55,6 @@ if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then
 fi
 
 mkdir -p "$OUTDIR"
-cat << EOF > "$OUTDIR"/inputs.SHA256SUMS
-$(sha256sum "$UNSIGNED_TARBALL" | cut -d' ' -f1)  inputs/$(basename "$UNSIGNED_TARBALL")
-$(sha256sum "$CODESIGNATURE_GIT_ARCHIVE" | cut -d' ' -f1)  inputs/$(basename "$CODESIGNATURE_GIT_ARCHIVE")
-EOF
 
 mkdir -p "$DISTSRC"
 (
@@ -103,3 +99,14 @@ mkdir -p "$DISTSRC"
 rm -rf "$ACTUAL_OUTDIR"
 mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \
     || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 )
+
+(
+    cd /outdir-base
+    {
+        echo "$UNSIGNED_TARBALL"
+        echo "$CODESIGNATURE_GIT_ARCHIVE"
+        find "$ACTUAL_OUTDIR" -type f
+    } | xargs realpath --relative-base="$PWD" \
+        | xargs sha256sum \
+        | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part
+)
diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm
index 12eab27a3e..d6fc7ffd84 100644
--- a/contrib/guix/manifest.scm
+++ b/contrib/guix/manifest.scm
@@ -22,6 +22,7 @@
              (gnu packages linux)
              (gnu packages llvm)
              (gnu packages mingw)
+             (gnu packages moreutils)
              (gnu packages perl)
              (gnu packages pkg-config)
              (gnu packages python)
@@ -572,6 +573,7 @@ inspecting signatures in Mach-O binaries.")
         patch
         gawk
         sed
+        moreutils
         ;; Compression and archiving
         tar
         bzip2