From 28a9c9b83924f585b397f0f3b8e9e73780ac0ad6 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 25 May 2021 23:51:11 -0400 Subject: Make SHA256SUMS fragment right after build --- contrib/guix/libexec/build.sh | 23 ++++++++++------------- contrib/guix/libexec/codesign.sh | 15 +++++++++++---- contrib/guix/manifest.scm | 2 ++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 3073b41baf..6741328473 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -230,20 +230,7 @@ if [ ! -e "$GIT_ARCHIVE" ]; then git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi -# tmpdir="$(mktemp -d)" -# ( -# cd "$tmpdir" -# mkdir -p inputs -# ln -sf --target-directory=inputs "$GIT_ARCHIVE" - -# mkdir -p "$OUTDIR" -# find -L inputs -type f -print0 | xargs -0 sha256sum > "${OUTDIR}/inputs.SHA256SUMS" -# ) - mkdir -p "$OUTDIR" -cat << EOF > "$OUTDIR"/inputs.SHA256SUMS -$(sha256sum "$GIT_ARCHIVE" | cut -d' ' -f1) inputs/$(basename "$GIT_ARCHIVE") -EOF ########################### # Binary Tarball Building # @@ -450,3 +437,13 @@ mkdir -p "$DISTSRC" rm -rf "$ACTUAL_OUTDIR" mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 1822da7ca4..b1eec686ec 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -55,10 +55,6 @@ if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then fi mkdir -p "$OUTDIR" -cat << EOF > "$OUTDIR"/inputs.SHA256SUMS -$(sha256sum "$UNSIGNED_TARBALL" | cut -d' ' -f1) inputs/$(basename "$UNSIGNED_TARBALL") -$(sha256sum "$CODESIGNATURE_GIT_ARCHIVE" | cut -d' ' -f1) inputs/$(basename "$CODESIGNATURE_GIT_ARCHIVE") -EOF mkdir -p "$DISTSRC" ( @@ -103,3 +99,14 @@ mkdir -p "$DISTSRC" rm -rf "$ACTUAL_OUTDIR" mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$UNSIGNED_TARBALL" + echo "$CODESIGNATURE_GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index d2bc789b60..efeeede9db 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -22,6 +22,7 @@ (gnu packages linux) (gnu packages llvm) (gnu packages mingw) + (gnu packages moreutils) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) @@ -572,6 +573,7 @@ inspecting signatures in Mach-O binaries.") patch gawk sed + moreutils ;; Compression and archiving tar bzip2 -- cgit v1.2.3 From 4cc35daed557f38b080360a89036b2e97a6f78c2 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 7 Jun 2021 15:05:33 -0400 Subject: Rewrite guix-{attest,verify} for new hier --- contrib/guix/guix-attest | 133 ++++++++++++++++++++++------------------------- contrib/guix/guix-verify | 109 ++++++++++++++++++++++++-------------- 2 files changed, 132 insertions(+), 110 deletions(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 081d1c0465..7757d4bd28 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -99,24 +99,34 @@ fi # We should be able to find at least one output ################ -echo "Looking for build output directories in ${OUTDIR_BASE}" +echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" shopt -s nullglob -OUTDIRS=( "${OUTDIR_BASE}"/* ) # This expands to an array of directories... +OUTDIRS=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... shopt -u nullglob +noncodesigned_fragments=() +codesigned_fragments=() + if (( ${#OUTDIRS[@]} )); then - echo "Found build output directories:" + echo "Found build output SHA256SUMS fragments:" for outdir in "${OUTDIRS[@]}"; do echo " '$outdir'" + case "$outdir" in + "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$outdir") + ;; + *) + noncodesigned_fragments+=("$outdir") + ;; + esac done echo else - echo "ERR: Could not find any build output directories in ${OUTDIR_BASE}" + echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" exit 1 fi - ############## ## Attest ## ############## @@ -126,82 +136,65 @@ fi # HOST: The output directory being attested # out_name() { - basename "$1" -} - -# Usage: out_sig_dir $outdir -# -# outdir: The output directory being attested -# -out_sig_dir() { - echo "$GUIX_SIGS_REPO/$VERSION/$(out_name "$1")/$signer_name" + basename "$(dirname "$1")" } -# Accumulate a list of signature directories that already exist... -outdirs_already_attested_to=() - echo "Attesting to build outputs for version: '${VERSION}'" echo "" -# MAIN LOGIC: Loop through each output for VERSION and attest to output in -# GUIX_SIGS_REPO as SIGNER, if attestation does not exist -for outdir in "${OUTDIRS[@]}"; do - if [ -e "${outdir}/SKIPATTEST.TAG" ]; then - echo "${outname}: SKIPPING: Output directory marked with SKIPATTEST.TAG file" - continue - fi - outname="$(out_name "$outdir")" - outsigdir="$(out_sig_dir "$outdir")" - if [ -e "$outsigdir" ]; then - echo "${outname}: SKIPPING: Signature directory already exists in the specified guix.sigs repository" - outdirs_already_attested_to+=("$outdir") +outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name" +mkdir -p "$outsigdir" +( + cd "$outsigdir" + + if [ -e "noncodesigned.SHA256SUMS" ]; then + echo "noncodesigned.SHA256SUMS already exists, using..." + elif (( ${#noncodesigned_fragments[@]} )); then + cat "${noncodesigned_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + > noncodesigned.SHA256SUMS else - # Clean up incomplete sigdir if something fails (likely gpg) - trap 'rm -rf "$outsigdir"' ERR - - mkdir -p "$outsigdir" - - ( - cd "$outdir" - - if [ -e inputs.SHA256SUMS ]; then - echo "${outname}: Including existent input SHA256SUMS" - cat inputs.SHA256SUMS >> "$outsigdir"/SHA256SUMS - fi + echo "no noncodesigned outputs found" + fi - echo "${outname}: Hashing build outputs to produce SHA256SUMS" - files="$(find -L . -type f ! -iname '*.SHA256SUMS')" - if [ -n "$files" ]; then - cut -c3- <<< "$files" | env LC_ALL=C sort | xargs sha256sum >> "$outsigdir"/SHA256SUMS - else - echo "ERR: ${outname}: No outputs found in '${outdir}'" - exit 1 - fi - ) - if [ -z "$NO_SIGN" ]; then - echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" - gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS + if [ -e noncodesigned.SHA256SUMS ]; then + # noncodesigned.SHA256SUMS already exists, or was produced, let's sanity + # check it. + ( cd "$OUTDIR_BASE"; sha256sum -c "$outsigdir"/noncodesigned.SHA256SUMS ) + + # Now produce all.SHA256SUMS manifest + if [ -e "all.SHA256SUMS" ]; then + echo "all.SHA256SUMS already there!" + elif (( ${#codesigned_fragments[@]} )); then + cat "${OUTDIRS[@]}" \ + | sort -u \ + | sort -k2 \ + > all.SHA256SUMS else - echo "${outname}: Not signing SHA256SUMS as \$NO_SIGN is not empty" + echo "no codesigned outputs found" fi - echo "" - trap - ERR # Reset ERR trap + if [ -e all.SHA256SUMS ]; then + ( cd "$OUTDIR_BASE"; sha256sum -c "$outsigdir"/all.SHA256SUMS ) + fi fi -done - -if (( ${#outdirs_already_attested_to[@]} )); then -# ...so that we can print them out nicely in a warning message -cat << EOF -WARN: Signature directories from '$signer_name' already exist in the specified - guix.sigs repository for the following output directories and were - skipped: -EOF -for outdir in "${outdirs_already_attested_to[@]}"; do - echo " '${outdir}'" - echo " Corresponds to: '$(out_sig_dir "$outdir")'" + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + gpg --detach-sign \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi echo "" -done -fi +) diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify index 629050956c..a6e2c4065e 100755 --- a/contrib/guix/guix-verify +++ b/contrib/guix/guix-verify @@ -56,58 +56,87 @@ cmd_usage exit 1 fi -################ -# We should be able to find at least one output -################ +############## +## Verify ## +############## OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" -echo "Looking for output signature directories in '${OUTSIGDIR_BASE}'" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} shopt -s nullglob -OUTSIGDIRS=( "$OUTSIGDIR_BASE"/* ) # This expands to an array of directories... +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) shopt -u nullglob -if (( ${#OUTSIGDIRS[@]} )); then - echo "Found output signature directories:" - for outsigdir in "${OUTSIGDIRS[@]}"; do - echo " '$outsigdir'" +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" done - echo + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" else - echo "ERR: Could not find any output signature directories in ${OUTSIGDIR_BASE}" - exit 1 + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" fi +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob -############## -## Verify ## -############## +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" -# MAIN LOGIC: Loop through each output for VERSION and check that the SHA256SUMS -# and SHA256SUMS.asc file match between signers, using the first -# available signer as the arbitrary comparison base. -for outsigdir in "${OUTSIGDIRS[@]}"; do - echo "BEGIN: Checking output signatures for $(basename "$outsigdir")" - echo "" - signer_dirs=( "$outsigdir"/* ) # This expands to an array of directories... - compare_signer_dir="${signer_dirs[0]}" # ...we just want the first one - for current_signer_dir in "${signer_dirs[@]}"; do - if ! gpg --quiet --batch --verify "$current_signer_dir"/SHA256SUMS.asc "$current_signer_dir"/SHA256SUMS; then - echo "ERR: Failed to verify GPG signature in '${current_signer_dir}/SHA256SUMS.asc'" - echo "" - echo "Hint: Either the signature is invalid or the public key is missing" - echo "" - elif ! diff --report-identical "$compare_signer_dir"/SHA256SUMS "$current_signer_dir"/SHA256SUMS; then - echo "ERR: The SHA256SUMS attestation in these two directories differ:" - echo " '${compare_signer_dir}'" - echo " '${current_signer_dir}'" - echo "" - else - echo "Verified: '${current_signer_dir}'" - echo "" - fi + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" done - echo "DONE: Checking output signatures for $(basename "$outsigdir")" + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" echo "" -done + exit 1 +fi -- cgit v1.2.3 From e2c40a4ed5272d72fea997bd936fba28bb753226 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 14 Jun 2021 17:00:26 -0400 Subject: guix-attest: Error out if SHA256SUMS is unexpected --- contrib/guix/guix-attest | 94 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 7757d4bd28..c8cf73d400 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -102,15 +102,15 @@ fi echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" shopt -s nullglob -OUTDIRS=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... shopt -u nullglob noncodesigned_fragments=() codesigned_fragments=() -if (( ${#OUTDIRS[@]} )); then +if (( ${#sha256sum_fragments[@]} )); then echo "Found build output SHA256SUMS fragments:" - for outdir in "${OUTDIRS[@]}"; do + for outdir in "${sha256sum_fragments[@]}"; do echo " '$outdir'" case "$outdir" in "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) @@ -139,6 +139,26 @@ out_name() { basename "$(dirname "$1")" } +shasum_already_exists() { +cat < noncodesigned.SHA256SUMS + > "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi else - echo "no noncodesigned outputs found" + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 fi - if [ -e noncodesigned.SHA256SUMS ]; then - # noncodesigned.SHA256SUMS already exists, or was produced, let's sanity - # check it. - ( cd "$OUTDIR_BASE"; sha256sum -c "$outsigdir"/noncodesigned.SHA256SUMS ) - - # Now produce all.SHA256SUMS manifest - if [ -e "all.SHA256SUMS" ]; then - echo "all.SHA256SUMS already there!" - elif (( ${#codesigned_fragments[@]} )); then - cat "${OUTDIRS[@]}" \ - | sort -u \ - | sort -k2 \ - > all.SHA256SUMS - else - echo "no codesigned outputs found" - fi + temp_codesigned="$(mktemp)" + trap 'rm -rf -- "$temp_codesigned"' EXIT - if [ -e all.SHA256SUMS ]; then - ( cd "$OUTDIR_BASE"; sha256sum -c "$outsigdir"/all.SHA256SUMS ) + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + > "$temp_codesigned" + if [ -e codesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_codesigned"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_codesigned" codesigned.SHA256SUMS fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." fi - if [ -z "$NO_SIGN" ]; then echo "Signing SHA256SUMS to produce SHA256SUMS.asc" for i in *.SHA256SUMS; do -- cgit v1.2.3