Merge #19205: script: previous_release.sh rewritten in python

9c34aff39309b8adc99d347e07b6ddb5366498e9 Remove previous_release.sh (Brian Liotti)
e1e5960e10a9329d9f55a3967d546ffbdd896030 script: Add previous_release.py (Brian Liotti)

Pull request description:

  Closes #18132

  Added functionality:
  1) checks file hash before untarring when using the binary download option

ACKs for top commit:
  fjahr:
    re-ACK 9c34aff39309b8adc99d347e07b6ddb5366498e9
  Sjors:
    tACK 9c34aff39309b8adc99d347e07b6ddb5366498e9

Tree-SHA512: 323f11828736a372a47f048592de8b027ddcd75b38f312dfc73f7b495d1e078bfeb384d9cdf434b3e70f2c6c0ce2da2df48e9a6460ac0e1967c6829a411c52d5

2 files changed, 222 insertions, 152 deletions

diff --git a/contrib/devtools/previous_release.py b/contrib/devtools/previous_release.py
new file mode 100755
index 0000000000..bea4f56010
--- /dev/null
+++ b/contrib/devtools/previous_release.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Build previous releases.
+
+import argparse
+import contextlib
+from fnmatch import fnmatch
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import hashlib
+
+
+@contextlib.contextmanager
+def pushd(new_dir) -> None:
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def download_binary(tag, args) -> int:
+    if Path(tag).is_dir():
+        if not args.remove_dir:
+            print('Using cached {}'.format(tag))
+            return 0
+        shutil.rmtree(tag)
+    Path(tag).mkdir()
+    bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
+    match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
+    if match:
+        bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
+            match.group(1), match.group(2))
+    tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
+        tag=tag[1:], platform=args.platform)
+    sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
+    tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
+        bin_path=bin_path, tarball=tarball)
+    sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
+        bin_path=bin_path)
+
+    print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
+    print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
+
+    header, status = subprocess.Popen(
+        ['curl', '-I', tarballUrl], stdout=subprocess.PIPE).communicate()
+    if re.search("404 Not Found", header.decode("utf-8")):
+        print("Binary tag was not found")
+        return 1
+
+    curlCmds = [
+        ['curl', '-O', tarballUrl],
+        ['curl', "-o", sha256Sums, sha256SumsUrl],
+    ]
+
+    for cmd in curlCmds:
+        ret = subprocess.run(cmd).returncode
+        if ret:
+            return ret
+
+    hasher = hashlib.sha256()
+    with open(tarball, "rb") as afile:
+        buf = afile.read()
+        hasher.update(buf)
+    afile.close()
+    tarballHash = hasher.hexdigest()
+    file = open(sha256Sums, 'r', encoding="utf-8")
+    lst = list(file.readlines())
+    file.close()
+    lastline = lst[len(lst)-1]
+
+    for line in lst:
+        if re.search(tarballHash, line):
+            print("Checksum matched")
+            break
+        elif lastline == line:
+            print("Checksum did not match")
+            Path(tarball).unlink()
+            return 1
+
+    # Bitcoin Core Release Signing Keys v0.11.0+
+    signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
+
+    isKeyPresent = subprocess.run(
+        ["gpg", "--list-keys", signingKey]).returncode
+    if isKeyPresent:
+        return isKeyPresent
+
+    isVerified = subprocess.run(
+        ["gpg", "--verify", sha256Sums]).returncode
+    if isVerified:
+        return isVerified
+
+    # Extract tarball
+    ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
+                          '--strip-components=1',
+                          'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
+    if ret:
+        return ret
+
+    Path(tarball).unlink()
+    Path(sha256Sums).unlink()
+    return 0
+
+
+def build_release(tag, args) -> int:
+    githubUrl = "https://github.com/bitcoin/bitcoin"
+    if args.remove_dir:
+        if Path(tag).is_dir():
+            shutil.rmtree(tag)
+    if not Path(tag).is_dir():
+        # fetch new tags
+        subprocess.run(
+            ["git", "fetch", githubUrl, "--tags"])
+        output = subprocess.check_output(['git', 'tag', '-l', tag])
+        if not output:
+            print('Tag {} not found'.format(tag))
+            return 1
+    ret = subprocess.run([
+        'git', 'clone', githubUrl, tag
+    ]).returncode
+    if ret:
+        return ret
+    with pushd(tag):
+        ret = subprocess.run(['git', 'checkout', tag]).returncode
+        if ret:
+            return ret
+        host = args.host
+        if args.depends:
+            with pushd('depends'):
+                ret = subprocess.run(['make', 'NO_QT=1']).returncode
+                if ret:
+                    return ret
+                host = os.environ.get(
+                    'HOST', subprocess.check_output(['./config.guess']))
+        config_flags = '--prefix={pwd}/depends/{host} '.format(
+            pwd=os.getcwd(),
+            host=host) + args.config_flags
+        cmds = [
+            './autogen.sh',
+            './configure {}'.format(config_flags),
+            'make',
+        ]
+        for cmd in cmds:
+            ret = subprocess.run(cmd.split()).returncode
+            if ret:
+                return ret
+        # Move binaries, so they're in the same place as in the
+        # release download
+        Path('bin').mkdir(exist_ok=True)
+        files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx']
+        for f in files:
+            Path('src/'+f).rename('bin/'+f)
+    return 0
+
+
+def check_host(args) -> int:
+    args.host = os.environ.get('HOST', subprocess.check_output(
+        './depends/config.guess').decode())
+    if args.download_binary:
+        platforms = {
+            'x86_64-*-linux*': 'x86_64-linux-gnu',
+            'x86_64-apple-darwin*': 'osx64',
+        }
+        args.platform = ''
+        for pattern, target in platforms.items():
+            if fnmatch(args.host, pattern):
+                args.platform = target
+        if not args.platform:
+            print('Not sure which binary to download for {}'.format(args.host))
+            return 1
+    return 0
+
+
+def main(args) -> int:
+    if not Path(args.target_dir).is_dir():
+        Path(args.target_dir).mkdir(exist_ok=True, parents=True)
+    print("Releases directory: {}".format(args.target_dir))
+    ret = check_host(args)
+    if ret:
+        return ret
+    if args.download_binary:
+        with pushd(args.target_dir):
+            for tag in args.tags:
+                ret = download_binary(tag, args)
+                if ret:
+                    return ret
+        return 0
+    args.config_flags = os.environ.get('CONFIG_FLAGS', '')
+    args.config_flags += ' --without-gui --disable-tests --disable-bench'
+    with pushd(args.target_dir):
+        for tag in args.tags:
+            ret = build_release(tag, args)
+            if ret:
+                return ret
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-r', '--remove-dir', action='store_true',
+                        help='remove existing directory.')
+    parser.add_argument('-d', '--depends', action='store_true',
+                        help='use depends.')
+    parser.add_argument('-b', '--download-binary', action='store_true',
+                        help='download release binary.')
+    parser.add_argument('-t', '--target-dir', action='store',
+                        help='target directory.', default='releases')
+    parser.add_argument('tags', nargs='+',
+                        help="release tags. e.g.: v0.18.1 v0.20.0rc2")
+    args = parser.parse_args()
+    sys.exit(main(args))
diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh
deleted file mode 100755
index d375291f47..0000000000
--- a/contrib/devtools/previous_release.sh
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Build previous releases.
-
-export LC_ALL=C
-
-CONFIG_FLAGS=""
-FUNCTIONAL_TESTS=0
-DELETE_EXISTING=0
-USE_DEPENDS=0
-DOWNLOAD_BINARY=0
-CONFIG_FLAGS=""
-TARGET="releases"
-
-while getopts ":hfrdbt:" opt; do
-  case $opt in
-    h)
-      echo "Usage: .previous_release.sh [options] tag1 tag2"
-      echo "  options:"
-      echo "  -h   Print this message"
-      echo "  -f   Configure for functional tests"
-      echo "  -r   Remove existing directory"
-      echo "  -d   Use depends"
-      echo "  -b   Download release binary"
-      echo "  -t   Target directory (default: releases)"
-      exit 0
-      ;;
-    f)
-      FUNCTIONAL_TESTS=1
-      CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench"
-      ;;
-    r)
-      DELETE_EXISTING=1
-      ;;
-    d)
-      USE_DEPENDS=1
-      ;;
-    b)
-      DOWNLOAD_BINARY=1
-      ;;
-    t)
-      TARGET=$OPTARG
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      exit 1
-      ;;
-  esac
-done
-
-shift $((OPTIND-1))
-
-if [ -z "$1" ]; then
-  echo "Specify release tag(s), e.g.: .previous_release v0.15.1"
-  exit 1
-fi
-
-if [ ! -d "$TARGET" ]; then
-  mkdir -p $TARGET
-fi
-
-if [ "$DOWNLOAD_BINARY" -eq "1" ]; then
-  HOST="${HOST:-$(./depends/config.guess)}"
-  case "$HOST" in
-    x86_64-*-linux*)
-      PLATFORM=x86_64-linux-gnu
-      ;;
-    x86_64-apple-darwin*)
-      PLATFORM=osx64
-      ;;
-    *)
-      echo "Not sure which binary to download for $HOST."
-      exit 1
-      ;;
-  esac
-fi
-
-echo "Releases directory: $TARGET"
-pushd "$TARGET" || exit 1
-{
-  for tag in "$@"
-  do
-    if [ "$DELETE_EXISTING" -eq "1" ]; then
-      if [ -d "$tag" ]; then
-        rm -r "$tag"
-      fi
-    fi
-
-    if [ "$DOWNLOAD_BINARY" -eq "0" ]; then
-
-      if [ ! -d "$tag" ]; then
-        if [ -z $(git tag -l "$tag") ]; then
-          echo "Tag $tag not found"
-          exit 1
-        fi
-
-        git clone https://github.com/bitcoin/bitcoin "$tag"
-        pushd "$tag" || exit 1
-        {
-          git checkout "$tag"
-          if [ "$USE_DEPENDS" -eq "1" ]; then
-            pushd depends || exit 1
-            {
-              if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then
-                make NO_QT=1
-              else
-                make
-              fi
-              HOST="${HOST:-$(./config.guess)}"
-            }
-            popd || exit 1
-            CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS"
-          fi
-          ./autogen.sh
-          ./configure $CONFIG_FLAGS
-          make
-          # Move binaries, so they're in the same place as in the release download:
-          mkdir bin
-          mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin
-          if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then
-            mv src/qt/bitcoin-qt bin
-          fi
-        }
-        popd || exit 1
-      fi
-    else
-      if [ -d "$tag" ]; then
-        echo "Using cached $tag"
-      else
-        mkdir "$tag"
-        if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then
-            BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}"
-        else
-            BIN_PATH="bin/bitcoin-core-${tag:1}"
-        fi
-        URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz"
-        echo "Fetching: $URL"
-        if ! curl -O -f $URL; then
-            echo "Download failed."
-            exit 1
-        fi
-        tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}"
-        rm "bitcoin-${tag:1}-$PLATFORM.tar.gz"
-      fi
-    fi
-  done
-}
-popd || exit 1