Merge #20689: contrib: replace binary verification script verify.sh with python rewrite

c86b9a65eb0d6d1e659415880702c4dc889c34e6 contrib: remove verify.sh (Sebastian Falbesoner)
c84838e7afb7b084a56a75e98325563b6de83124 contrib: binary verification script verify.sh rewritten in python (Sebastian Falbesoner)

Pull request description:

  The rationale for the PR is the same as for #18132:
  > Most of our test scripts are written in python. We don't have enough reviewers for bash scripts and they tend to be clumsy anyway. Especially when it comes to argument parsing.

  Note that there are still a lot of things that could be improved in this replacement (e.g. using regexps for version string parsing, adding type annotations, dividing up into more functions, getting a pylint score closer to 10, etc.), but I found the original shell script quite hard to read, so it's possibly still a good first step for an improvement.
  ~Not sure though if it's worth the reviewers time, and if it's even continued to be used long-term (maybe there are plans to merge it with `get_previous_releases.py`, which partly does the same?), so chasing for Concept ACKs right now.~

ACKs for top commit:
  laanwj:
    Tested and code review ACK c86b9a65eb0d6d1e659415880702c4dc889c34e6

Tree-SHA512: f7949eead4ef7e5913fe273923ae5c5299408db485146cf996cdf6f8ad8c0ee4f4b30bb6b08a5964000d97b2ae2e7a1bdc88d11c613c16d2d135d80b444e3b16

3 files changed, 190 insertions, 184 deletions

diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md
index 4209fdb364..c50d4bef71 100644
--- a/contrib/verifybinaries/README.md
+++ b/contrib/verifybinaries/README.md
@@ -21,21 +21,21 @@ The script returns 0 if everything passes the checks. It returns 1 if either the
 
 
 ```sh
-./verify.sh bitcoin-core-0.11.2
-./verify.sh bitcoin-core-0.12.0
-./verify.sh bitcoin-core-0.13.0-rc3
+./verify.py bitcoin-core-0.11.2
+./verify.py bitcoin-core-0.12.0
+./verify.py bitcoin-core-0.13.0-rc3
 ```
 
 If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.:
 
 ```sh
-./verify.sh bitcoin-core-0.11.2-osx
-./verify.sh 0.12.0-linux
-./verify.sh bitcoin-core-0.13.0-rc3-win64
+./verify.py bitcoin-core-0.11.2-osx
+./verify.py 0.12.0-linux
+./verify.py bitcoin-core-0.13.0-rc3-win64
 ```
 
 If you do not want to keep the downloaded binaries, specify anything as the second parameter.
 
 ```sh
-./verify.sh bitcoin-core-0.13.0 delete
+./verify.py bitcoin-core-0.13.0 delete
 ```
diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py
new file mode 100755
index 0000000000..b97ad11b8e
--- /dev/null
+++ b/contrib/verifybinaries/verify.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Script for verifying Bitoin Core release binaries
+
+This script attempts to download the signature file SHA256SUMS.asc from
+bitcoincore.org and bitcoin.org and compares them.
+It first checks if the signature passes, and then downloads the files
+specified in the file, and checks if the hashes of these files match those
+that are specified in the signature file.
+The script returns 0 if everything passes the checks. It returns 1 if either
+the signature check or the hash check doesn't pass. If an error occurs the
+return value is >= 2.
+"""
+from hashlib import sha256
+import os
+import subprocess
+import sys
+from textwrap import indent
+
+WORKINGDIR = "/tmp/bitcoin_verify_binaries"
+HASHFILE = "hashes.tmp"
+HOST1 = "https://bitcoincore.org"
+HOST2 = "https://bitcoin.org"
+VERSIONPREFIX = "bitcoin-core-"
+SIGNATUREFILENAME = "SHA256SUMS.asc"
+
+
+def parse_version_string(version_str):
+    if version_str.startswith(VERSIONPREFIX):  # remove version prefix
+        version_str = version_str[len(VERSIONPREFIX):]
+
+    parts = version_str.split('-')
+    version_base = parts[0]
+    version_rc = ""
+    version_os = ""
+    if len(parts) == 2:  # "<version>-rcN" or "version-platform"
+        if "rc" in parts[1]:
+            version_rc = parts[1]
+        else:
+            version_os = parts[1]
+    elif len(parts) == 3:  # "<version>-rcN-platform"
+        version_rc = parts[1]
+        version_os = parts[2]
+
+    return version_base, version_rc, version_os
+
+
+def download_with_wget(remote_file, local_file=None):
+    if local_file:
+        wget_args = ['wget', '-O', local_file, remote_file]
+    else:
+        # use timestamping mechanism if local filename is not explicitely set
+        wget_args = ['wget', '-N', remote_file]
+
+    result = subprocess.run(wget_args,
+                            stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    return result.returncode == 0, result.stdout.decode().rstrip()
+
+
+def files_are_equal(filename1, filename2):
+    with open(filename1, 'rb') as file1:
+        contents1 = file1.read()
+    with open(filename2, 'rb') as file2:
+        contents2 = file2.read()
+    return contents1 == contents2
+
+
+def verify_with_gpg(signature_filename, output_filename):
+    result = subprocess.run(['gpg', '--yes', '--decrypt', '--output',
+                             output_filename, signature_filename],
+                             stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    return result.returncode, result.stdout.decode().rstrip()
+
+
+def remove_files(filenames):
+    for filename in filenames:
+        os.remove(filename)
+
+
+def main(args):
+    # sanity check
+    if len(args) < 1:
+        print("Error: need to specify a version on the command line")
+        return 3
+
+    # determine remote dir dependend on provided version string
+    version_base, version_rc, os_filter = parse_version_string(args[0])
+    remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/"
+    if version_rc:
+        remote_dir += f"test.{version_rc}/"
+    remote_sigfile = remote_dir + SIGNATUREFILENAME
+
+    # create working directory
+    os.makedirs(WORKINGDIR, exist_ok=True)
+    os.chdir(WORKINGDIR)
+
+    # fetch first signature file
+    sigfile1 = SIGNATUREFILENAME
+    success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1)
+    if not success:
+        print("Error: couldn't fetch signature file. "
+              "Have you specified the version number in the following format?")
+        print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] "
+              f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)")
+        print("wget output:")
+        print(indent(output, '\t'))
+        return 4
+
+    # fetch second signature file
+    sigfile2 = SIGNATUREFILENAME + ".2"
+    success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2)
+    if not success:
+        print("bitcoin.org failed to provide signature file, "
+              "but bitcoincore.org did?")
+        print("wget output:")
+        print(indent(output, '\t'))
+        remove_files([sigfile1])
+        return 5
+
+    # ensure that both signature files are equal
+    if not files_are_equal(sigfile1, sigfile2):
+        print("bitcoin.org and bitcoincore.org signature files were not equal?")
+        print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}")
+        return 6
+
+    # check signature and extract data into file
+    retval, output = verify_with_gpg(sigfile1, HASHFILE)
+    if retval != 0:
+        if retval == 1:
+            print("Bad signature.")
+        elif retval == 2:
+            print("gpg error. Do you have the Bitcoin Core binary release "
+                  "signing key installed?")
+        print("gpg output:")
+        print(indent(output, '\t'))
+        remove_files([sigfile1, sigfile2, HASHFILE])
+        return 1
+
+    # extract hashes/filenames of binaries to verify from hash file;
+    # each line has the following format: "<hash> <binary_filename>"
+    with open(HASHFILE, 'r', encoding='utf8') as hash_file:
+        hashes_to_verify = [
+            line.split()[:2] for line in hash_file if os_filter in line]
+    remove_files([HASHFILE])
+    if not hashes_to_verify:
+        print("error: no files matched the platform specified")
+        return 7
+
+    # download binaries
+    for _, binary_filename in hashes_to_verify:
+        print(f"Downloading {binary_filename}")
+        download_with_wget(HOST1 + remote_dir + binary_filename)
+
+    # verify hashes
+    offending_files = []
+    for hash_expected, binary_filename in hashes_to_verify:
+        with open(binary_filename, 'rb') as binary_file:
+            hash_calculated = sha256(binary_file.read()).hexdigest()
+        if hash_calculated != hash_expected:
+            offending_files.append(binary_filename)
+    if offending_files:
+        print("Hashes don't match.")
+        print("Offending files:")
+        print('\n'.join(offending_files))
+        return 1
+    verified_binaries = [entry[1] for entry in hashes_to_verify]
+
+    # clean up files if desired
+    if len(args) >= 2:
+        print("Clean up the binaries")
+        remove_files([sigfile1, sigfile2] + verified_binaries)
+    else:
+        print(f"Keep the binaries in {WORKINGDIR}")
+
+    print("Verified hashes of")
+    print('\n'.join(verified_binaries))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh
deleted file mode 100755
index 4296998631..0000000000
--- a/contrib/verifybinaries/verify.sh
+++ /dev/null
@@ -1,177 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2016-2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-###   This script attempts to download the signature file SHA256SUMS.asc from
-###   bitcoincore.org and bitcoin.org and compares them.
-###   It first checks if the signature passes, and then downloads the files specified in
-###   the file, and checks if the hashes of these files match those that are specified
-###   in the signature file.
-###   The script returns 0 if everything passes the checks. It returns 1 if either the
-###   signature check or the hash check doesn't pass. If an error occurs the return value is 2
-
-export LC_ALL=C
-function clean_up {
-   for file in "$@"
-   do
-      rm "$file" 2> /dev/null
-   done
-}
-
-WORKINGDIR="/tmp/bitcoin_verify_binaries"
-TMPFILE="hashes.tmp"
-
-SIGNATUREFILENAME="SHA256SUMS.asc"
-RCSUBDIR="test"
-HOST1="https://bitcoincore.org"
-HOST2="https://bitcoin.org"
-BASEDIR="/bin/"
-VERSIONPREFIX="bitcoin-core-"
-RCVERSIONSTRING="rc"
-
-if [ ! -d "$WORKINGDIR" ]; then
-   mkdir "$WORKINGDIR"
-fi
-
-cd "$WORKINGDIR" || exit 1
-
-#test if a version number has been passed as an argument
-if [ -n "$1" ]; then
-   #let's also check if the version number includes the prefix 'bitcoin-',
-   #  and add this prefix if it doesn't
-   if [[ $1 == "$VERSIONPREFIX"* ]]; then
-      VERSION="$1"
-   else
-      VERSION="$VERSIONPREFIX$1"
-   fi
-
-   STRIPPEDLAST="${VERSION%-*}"
-
-   #now let's see if the version string contains "rc" or a platform name (e.g. "osx")
-   if [[ "$STRIPPEDLAST-" == "$VERSIONPREFIX" ]]; then
-      BASEDIR="$BASEDIR$VERSION/"
-   else
-      # let's examine the last part to see if it's rc and/or platform name
-      STRIPPEDNEXTTOLAST="${STRIPPEDLAST%-*}"
-      if [[ "$STRIPPEDNEXTTOLAST-" == "$VERSIONPREFIX" ]]; then
-
-         LASTSUFFIX="${VERSION##*-}"
-         VERSION="$STRIPPEDLAST"
-
-         if [[ $LASTSUFFIX == *"$RCVERSIONSTRING"* ]]; then
-            RCVERSION="$LASTSUFFIX"
-         else
-            PLATFORM="$LASTSUFFIX"
-         fi
-
-      else
-         RCVERSION="${STRIPPEDLAST##*-}"
-         PLATFORM="${VERSION##*-}"
-
-         VERSION="$STRIPPEDNEXTTOLAST"
-      fi
-
-      BASEDIR="$BASEDIR$VERSION/"
-      if [[ $RCVERSION == *"$RCVERSIONSTRING"* ]]; then
-         BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/"
-      fi
-   fi
-else
-   echo "Error: need to specify a version on the command line"
-   exit 2
-fi
-
-if ! WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1); then
-   echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?"
-   # shellcheck disable=SC1087
-   echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)"
-   echo "wget output:"
-   # shellcheck disable=SC2001
-   echo "$WGETOUT"|sed 's/^/\t/g'
-   exit 2
-fi
-
-if ! WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1); then
-   echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?"
-   echo "wget output:"
-   # shellcheck disable=SC2001
-   echo "$WGETOUT"|sed 's/^/\t/g'
-   clean_up $SIGNATUREFILENAME
-   exit 3
-fi
-
-SIGFILEDIFFS="$(diff $SIGNATUREFILENAME $SIGNATUREFILENAME.2)"
-if [ "$SIGFILEDIFFS" != "" ]; then
-   echo "bitcoin.org and bitcoincore.org signature files were not equal?"
-   clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2
-   exit 4
-fi
-
-#then we check it
-GPGOUT=$(gpg --yes --decrypt --output "$TMPFILE" "$SIGNATUREFILENAME" 2>&1)
-
-#return value 0: good signature
-#return value 1: bad signature
-#return value 2: gpg error
-
-RET="$?"
-if [ $RET -ne 0 ]; then
-   if [ $RET -eq 1 ]; then
-      #and notify the user if it's bad
-      echo "Bad signature."
-   elif [ $RET -eq 2 ]; then
-      #or if a gpg error has occurred
-      echo "gpg error. Do you have the Bitcoin Core binary release signing key installed?"
-   fi
-
-   echo "gpg output:"
-   # shellcheck disable=SC2001
-   echo "$GPGOUT"|sed 's/^/\t/g'
-   clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE
-   exit "$RET"
-fi
-
-if [ -n "$PLATFORM" ]; then
-   grep $PLATFORM $TMPFILE > "$TMPFILE-plat"
-   TMPFILESIZE=$(stat -c%s "$TMPFILE-plat")
-   if [ $TMPFILESIZE -eq 0 ]; then
-      echo "error: no files matched the platform specified" && exit 3
-   fi
-   mv "$TMPFILE-plat" $TMPFILE
-fi
-
-#here we extract the filenames from the signature file
-FILES=$(awk '{print $2}' "$TMPFILE")
-
-#and download these one by one
-for file in $FILES
-do
-   echo "Downloading $file"
-   wget --quiet -N "$HOST1$BASEDIR$file"
-done
-
-#check hashes
-DIFF=$(diff <(sha256sum $FILES) "$TMPFILE")
-
-if [ $? -eq 1 ]; then
-   echo "Hashes don't match."
-   echo "Offending files:"
-   echo "$DIFF"|grep "^<"|awk '{print "\t"$3}'
-   exit 1
-elif [ $? -gt 1 ]; then
-   echo "Error executing 'diff'"
-   exit 2
-fi
-
-if [ -n "$2" ]; then
-   echo "Clean up the binaries"
-   clean_up $FILES $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE
-else
-   echo "Keep the binaries in $WORKINGDIR"
-   clean_up $TMPFILE
-fi
-
-echo -e "Verified hashes of \n$FILES"
-
-exit 0