Merge #19205: script: previous_release.sh rewritten in python

9c34aff39309b8adc99d347e07b6ddb5366498e9 Remove previous_release.sh (Brian Liotti)
e1e5960e10a9329d9f55a3967d546ffbdd896030 script: Add previous_release.py (Brian Liotti)

Pull request description:

  Closes #18132

  Added functionality:
  1) checks file hash before untarring when using the binary download option

ACKs for top commit:
  fjahr:
    re-ACK 9c34aff39309b8adc99d347e07b6ddb5366498e9
  Sjors:
    tACK 9c34aff39309b8adc99d347e07b6ddb5366498e9

Tree-SHA512: 323f11828736a372a47f048592de8b027ddcd75b38f312dfc73f7b495d1e078bfeb384d9cdf434b3e70f2c6c0ce2da2df48e9a6460ac0e1967c6829a411c52d5

6 files changed, 226 insertions, 156 deletions

diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh
index 057395eba6..131ea21677 100755
--- a/ci/test/05_before_script.sh
+++ b/ci/test/05_before_script.sh
@@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then
 fi
 if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
   BEGIN_FOLD previous-versions
-  DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+  DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
   END_FOLD
 fi
diff --git a/contrib/devtools/previous_release.py b/contrib/devtools/previous_release.py
new file mode 100755
index 0000000000..bea4f56010
--- /dev/null
+++ b/contrib/devtools/previous_release.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Build previous releases.
+
+import argparse
+import contextlib
+from fnmatch import fnmatch
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import hashlib
+
+
+@contextlib.contextmanager
+def pushd(new_dir) -> None:
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def download_binary(tag, args) -> int:
+    if Path(tag).is_dir():
+        if not args.remove_dir:
+            print('Using cached {}'.format(tag))
+            return 0
+        shutil.rmtree(tag)
+    Path(tag).mkdir()
+    bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
+    match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
+    if match:
+        bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
+            match.group(1), match.group(2))
+    tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
+        tag=tag[1:], platform=args.platform)
+    sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
+    tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
+        bin_path=bin_path, tarball=tarball)
+    sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
+        bin_path=bin_path)
+
+    print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
+    print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
+
+    header, status = subprocess.Popen(
+        ['curl', '-I', tarballUrl], stdout=subprocess.PIPE).communicate()
+    if re.search("404 Not Found", header.decode("utf-8")):
+        print("Binary tag was not found")
+        return 1
+
+    curlCmds = [
+        ['curl', '-O', tarballUrl],
+        ['curl', "-o", sha256Sums, sha256SumsUrl],
+    ]
+
+    for cmd in curlCmds:
+        ret = subprocess.run(cmd).returncode
+        if ret:
+            return ret
+
+    hasher = hashlib.sha256()
+    with open(tarball, "rb") as afile:
+        buf = afile.read()
+        hasher.update(buf)
+    afile.close()
+    tarballHash = hasher.hexdigest()
+    file = open(sha256Sums, 'r', encoding="utf-8")
+    lst = list(file.readlines())
+    file.close()
+    lastline = lst[len(lst)-1]
+
+    for line in lst:
+        if re.search(tarballHash, line):
+            print("Checksum matched")
+            break
+        elif lastline == line:
+            print("Checksum did not match")
+            Path(tarball).unlink()
+            return 1
+
+    # Bitcoin Core Release Signing Keys v0.11.0+
+    signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
+
+    isKeyPresent = subprocess.run(
+        ["gpg", "--list-keys", signingKey]).returncode
+    if isKeyPresent:
+        return isKeyPresent
+
+    isVerified = subprocess.run(
+        ["gpg", "--verify", sha256Sums]).returncode
+    if isVerified:
+        return isVerified
+
+    # Extract tarball
+    ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
+                          '--strip-components=1',
+                          'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
+    if ret:
+        return ret
+
+    Path(tarball).unlink()
+    Path(sha256Sums).unlink()
+    return 0
+
+
+def build_release(tag, args) -> int:
+    githubUrl = "https://github.com/bitcoin/bitcoin"
+    if args.remove_dir:
+        if Path(tag).is_dir():
+            shutil.rmtree(tag)
+    if not Path(tag).is_dir():
+        # fetch new tags
+        subprocess.run(
+            ["git", "fetch", githubUrl, "--tags"])
+        output = subprocess.check_output(['git', 'tag', '-l', tag])
+        if not output:
+            print('Tag {} not found'.format(tag))
+            return 1
+    ret = subprocess.run([
+        'git', 'clone', githubUrl, tag
+    ]).returncode
+    if ret:
+        return ret
+    with pushd(tag):
+        ret = subprocess.run(['git', 'checkout', tag]).returncode
+        if ret:
+            return ret
+        host = args.host
+        if args.depends:
+            with pushd('depends'):
+                ret = subprocess.run(['make', 'NO_QT=1']).returncode
+                if ret:
+                    return ret
+                host = os.environ.get(
+                    'HOST', subprocess.check_output(['./config.guess']))
+        config_flags = '--prefix={pwd}/depends/{host} '.format(
+            pwd=os.getcwd(),
+            host=host) + args.config_flags
+        cmds = [
+            './autogen.sh',
+            './configure {}'.format(config_flags),
+            'make',
+        ]
+        for cmd in cmds:
+            ret = subprocess.run(cmd.split()).returncode
+            if ret:
+                return ret
+        # Move binaries, so they're in the same place as in the
+        # release download
+        Path('bin').mkdir(exist_ok=True)
+        files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx']
+        for f in files:
+            Path('src/'+f).rename('bin/'+f)
+    return 0
+
+
+def check_host(args) -> int:
+    args.host = os.environ.get('HOST', subprocess.check_output(
+        './depends/config.guess').decode())
+    if args.download_binary:
+        platforms = {
+            'x86_64-*-linux*': 'x86_64-linux-gnu',
+            'x86_64-apple-darwin*': 'osx64',
+        }
+        args.platform = ''
+        for pattern, target in platforms.items():
+            if fnmatch(args.host, pattern):
+                args.platform = target
+        if not args.platform:
+            print('Not sure which binary to download for {}'.format(args.host))
+            return 1
+    return 0
+
+
+def main(args) -> int:
+    if not Path(args.target_dir).is_dir():
+        Path(args.target_dir).mkdir(exist_ok=True, parents=True)
+    print("Releases directory: {}".format(args.target_dir))
+    ret = check_host(args)
+    if ret:
+        return ret
+    if args.download_binary:
+        with pushd(args.target_dir):
+            for tag in args.tags:
+                ret = download_binary(tag, args)
+                if ret:
+                    return ret
+        return 0
+    args.config_flags = os.environ.get('CONFIG_FLAGS', '')
+    args.config_flags += ' --without-gui --disable-tests --disable-bench'
+    with pushd(args.target_dir):
+        for tag in args.tags:
+            ret = build_release(tag, args)
+            if ret:
+                return ret
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-r', '--remove-dir', action='store_true',
+                        help='remove existing directory.')
+    parser.add_argument('-d', '--depends', action='store_true',
+                        help='use depends.')
+    parser.add_argument('-b', '--download-binary', action='store_true',
+                        help='download release binary.')
+    parser.add_argument('-t', '--target-dir', action='store',
+                        help='target directory.', default='releases')
+    parser.add_argument('tags', nargs='+',
+                        help="release tags. e.g.: v0.18.1 v0.20.0rc2")
+    args = parser.parse_args()
+    sys.exit(main(args))
diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh
deleted file mode 100755
index d375291f47..0000000000
--- a/contrib/devtools/previous_release.sh
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Build previous releases.
-
-export LC_ALL=C
-
-CONFIG_FLAGS=""
-FUNCTIONAL_TESTS=0
-DELETE_EXISTING=0
-USE_DEPENDS=0
-DOWNLOAD_BINARY=0
-CONFIG_FLAGS=""
-TARGET="releases"
-
-while getopts ":hfrdbt:" opt; do
-  case $opt in
-    h)
-      echo "Usage: .previous_release.sh [options] tag1 tag2"
-      echo "  options:"
-      echo "  -h   Print this message"
-      echo "  -f   Configure for functional tests"
-      echo "  -r   Remove existing directory"
-      echo "  -d   Use depends"
-      echo "  -b   Download release binary"
-      echo "  -t   Target directory (default: releases)"
-      exit 0
-      ;;
-    f)
-      FUNCTIONAL_TESTS=1
-      CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench"
-      ;;
-    r)
-      DELETE_EXISTING=1
-      ;;
-    d)
-      USE_DEPENDS=1
-      ;;
-    b)
-      DOWNLOAD_BINARY=1
-      ;;
-    t)
-      TARGET=$OPTARG
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      exit 1
-      ;;
-  esac
-done
-
-shift $((OPTIND-1))
-
-if [ -z "$1" ]; then
-  echo "Specify release tag(s), e.g.: .previous_release v0.15.1"
-  exit 1
-fi
-
-if [ ! -d "$TARGET" ]; then
-  mkdir -p $TARGET
-fi
-
-if [ "$DOWNLOAD_BINARY" -eq "1" ]; then
-  HOST="${HOST:-$(./depends/config.guess)}"
-  case "$HOST" in
-    x86_64-*-linux*)
-      PLATFORM=x86_64-linux-gnu
-      ;;
-    x86_64-apple-darwin*)
-      PLATFORM=osx64
-      ;;
-    *)
-      echo "Not sure which binary to download for $HOST."
-      exit 1
-      ;;
-  esac
-fi
-
-echo "Releases directory: $TARGET"
-pushd "$TARGET" || exit 1
-{
-  for tag in "$@"
-  do
-    if [ "$DELETE_EXISTING" -eq "1" ]; then
-      if [ -d "$tag" ]; then
-        rm -r "$tag"
-      fi
-    fi
-
-    if [ "$DOWNLOAD_BINARY" -eq "0" ]; then
-
-      if [ ! -d "$tag" ]; then
-        if [ -z $(git tag -l "$tag") ]; then
-          echo "Tag $tag not found"
-          exit 1
-        fi
-
-        git clone https://github.com/bitcoin/bitcoin "$tag"
-        pushd "$tag" || exit 1
-        {
-          git checkout "$tag"
-          if [ "$USE_DEPENDS" -eq "1" ]; then
-            pushd depends || exit 1
-            {
-              if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then
-                make NO_QT=1
-              else
-                make
-              fi
-              HOST="${HOST:-$(./config.guess)}"
-            }
-            popd || exit 1
-            CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS"
-          fi
-          ./autogen.sh
-          ./configure $CONFIG_FLAGS
-          make
-          # Move binaries, so they're in the same place as in the release download:
-          mkdir bin
-          mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin
-          if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then
-            mv src/qt/bitcoin-qt bin
-          fi
-        }
-        popd || exit 1
-      fi
-    else
-      if [ -d "$tag" ]; then
-        echo "Using cached $tag"
-      else
-        mkdir "$tag"
-        if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then
-            BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}"
-        else
-            BIN_PATH="bin/bitcoin-core-${tag:1}"
-        fi
-        URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz"
-        echo "Fetching: $URL"
-        if ! curl -O -f $URL; then
-            echo "Download failed."
-            exit 1
-        fi
-        tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}"
-        rm "bitcoin-${tag:1}-$PLATFORM.tar.gz"
-      fi
-    fi
-  done
-}
-popd || exit 1
diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py
index 5cddd6527e..07dd0f8f82 100755
--- a/test/functional/feature_backwards_compatibility.py
+++ b/test/functional/feature_backwards_compatibility.py
@@ -6,7 +6,7 @@
 
 Test various backwards compatibility scenarios. Download the previous node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
 Due to a hardfork in regtest, it can't be used to sync nodes.
diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py
index 999399dec0..31fb751904 100755
--- a/test/functional/mempool_compatibility.py
+++ b/test/functional/mempool_compatibility.py
@@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden
 In case we need to break mempool compatibility we can continue to use the test by just bumping the version number.
 
 Download node binaries:
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests.
 """
diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py
index cc2139a027..1a76f65215 100755
--- a/test/functional/wallet_upgradewallet.py
+++ b/test/functional/wallet_upgradewallet.py
@@ -6,7 +6,7 @@
 
 Test upgradewallet RPC. Download node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py
 """