Merge #19813: util, ci: Hard code previous release tarball checksums

0374e821bd9e9498ce9c03aa8e5435870019978b util: Hard code previous release tarball checksums (Hennadii Stepanov)
bd897ce79f72a44a2e609f95433e251a3fd9eb9c scripted-diff: Move previous_release.py to test/get_previous_releases.py (Hennadii Stepanov)

Pull request description:

  #19205 introduced signature verifying for the downloaded `SHA256SUMS.asc`.
  This approach is brittle and does not work in CI environment for many reasons:
  - https://github.com/bitcoin/bitcoin/issues/19812#issuecomment-680760663
  - https://github.com/bitcoin/bitcoin/pull/19013#discussion_r459590779

  This PR:
  - implements **Sjors**' [idea](https://github.com/bitcoin/bitcoin/pull/19205#pullrequestreview-426080048):
  > Alternatively we might as well hard code the checksum for each `tar.gz` release in the source code, here.

  - is an alternative to 5a2c31e528e6bd60635096f233252f3c717f366d (#19013)

  - fixes #19812

  - updates v0.17.1 to v0.17.2

ACKs for top commit:
  MarcoFalke:
    cr ACK 0374e821bd9e9498ce9c03aa8e5435870019978b
  Sjors:
    tACK 0374e821bd9e9498ce9c03aa8e5435870019978b

Tree-SHA512: cacdcf9f5209eae7da357abb3445585ad2f980920fd5bf75527ce89974d3f531a4cf8b5b35edfc116b23bfdfb45c0437cb14cbc416d76ed2dc5b9e6d33cdad71

6 files changed, 41 insertions, 30 deletions

diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh
index f9d869b4fd..6f2e39429c 100644
--- a/ci/test/00_setup_env_native_qt5.sh
+++ b/ci/test/00_setup_env_native_qt5.sh
@@ -15,5 +15,5 @@ export RUN_SECURITY_TESTS="true"
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
-export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1"
+export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1"
 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process"
diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh
index 131ea21677..8ce839fc04 100755
--- a/ci/test/05_before_script.sh
+++ b/ci/test/05_before_script.sh
@@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then
 fi
 if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
   BEGIN_FOLD previous-versions
-  DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+  DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
   END_FOLD
 fi
diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py
index 07dd0f8f82..126c9fe929 100755
--- a/test/functional/feature_backwards_compatibility.py
+++ b/test/functional/feature_backwards_compatibility.py
@@ -6,7 +6,7 @@
 
 Test various backwards compatibility scenarios. Download the previous node binaries:
 
-contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
 Due to a hardfork in regtest, it can't be used to sync nodes.
diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py
index 31fb751904..75ca8d3236 100755
--- a/test/functional/mempool_compatibility.py
+++ b/test/functional/mempool_compatibility.py
@@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden
 In case we need to break mempool compatibility we can continue to use the test by just bumping the version number.
 
 Download node binaries:
-contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests.
 """
diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py
index 1a76f65215..f3d6e74829 100755
--- a/test/functional/wallet_upgradewallet.py
+++ b/test/functional/wallet_upgradewallet.py
@@ -6,7 +6,7 @@
 
 Test upgradewallet RPC. Download node binaries:
 
-contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py
 """
diff --git a/contrib/devtools/previous_release.py b/test/get_previous_releases.py
index 5599051cf3..1348b8246b 100755
--- a/contrib/devtools/previous_release.py
+++ b/test/get_previous_releases.py
@@ -20,6 +20,40 @@ import sys
 import hashlib
 
 
+SHA256_SUMS = {
+"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz",
+"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz",
+"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz",
+"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz",
+"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz",
+
+"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz",
+"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz",
+"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz",
+"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz",
+"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz",
+
+"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz",
+"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz",
+"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz",
+"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz",
+"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz",
+
+"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz",
+"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz",
+"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz",
+"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz",
+"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz",
+"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz",
+
+"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz",
+"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz",
+"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz",
+"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz",
+"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz",
+"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz"
+}
+
 @contextlib.contextmanager
 def pushd(new_dir) -> None:
     previous_dir = os.getcwd()
@@ -44,14 +78,10 @@ def download_binary(tag, args) -> int:
             match.group(1), match.group(2))
     tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
         tag=tag[1:], platform=args.platform)
-    sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
     tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
         bin_path=bin_path, tarball=tarball)
-    sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
-        bin_path=bin_path)
 
     print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
-    print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
 
     header, status = subprocess.Popen(
         ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate()
@@ -60,8 +90,7 @@ def download_binary(tag, args) -> int:
         return 1
 
     curlCmds = [
-        ['curl', '--remote-name', tarballUrl],
-        ['curl', '--output', sha256Sums, sha256SumsUrl],
+        ['curl', '--remote-name', tarballUrl]
     ]
 
     for cmd in curlCmds:
@@ -73,29 +102,12 @@ def download_binary(tag, args) -> int:
     with open(tarball, "rb") as afile:
         hasher.update(afile.read())
     tarballHash = hasher.hexdigest()
-    tarballHash = '{}  {}\n'.format(tarballHash, tarball)
-    with open(sha256Sums, 'r', encoding="utf-8") as afile:
-        shasums = afile.readlines()
 
-    if tarballHash not in shasums:
+    if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash] != tarball:
         print("Checksum did not match")
-        Path(tarball).unlink()
         return 1
     print("Checksum matched")
 
-    # Bitcoin Core Release Signing Keys v0.11.0+
-    signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
-
-    isKeyPresent = subprocess.run(
-        ["gpg", "--list-keys", signingKey]).returncode
-    if isKeyPresent:
-        return isKeyPresent
-
-    isVerified = subprocess.run(
-        ["gpg", "--verify", sha256Sums]).returncode
-    if isVerified:
-        return isVerified
-
     # Extract tarball
     ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
                           '--strip-components=1',
@@ -104,7 +116,6 @@ def download_binary(tag, args) -> int:
         return ret
 
     Path(tarball).unlink()
-    Path(sha256Sums).unlink()
     return 0