diff options
author | fanquake <fanquake@gmail.com> | 2023-04-09 12:38:22 +0200 |
---|---|---|
committer | fanquake <fanquake@gmail.com> | 2023-04-09 12:43:59 +0200 |
commit | 663a89cfed5e924e79a7a4cb7f64d4c3181cc11d (patch) | |
tree | 3c38eb7e91fcf247be26fd9059014f9506860238 /contrib/verify-binaries | |
parent | db720b5a703c90625fa7a4773bd2db5672427cbe (diff) |
contrib: move verify scripts to verify-binaries
Diffstat (limited to 'contrib/verify-binaries')
-rw-r--r-- | contrib/verify-binaries/README.md | 88 | ||||
-rwxr-xr-x | contrib/verify-binaries/test.py | 59 | ||||
-rwxr-xr-x | contrib/verify-binaries/verify.py | 713 |
3 files changed, 860 insertions, 0 deletions
diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md new file mode 100644 index 0000000000..b29891bf07 --- /dev/null +++ b/contrib/verify-binaries/README.md @@ -0,0 +1,88 @@ +### Verify Binaries + +#### Preparation + +As of Bitcoin Core v22.0, releases are signed by a number of public keys on the basis +of the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs/). When +verifying binary downloads, you (the end user) decide which of these public keys you +trust and then use that trust model to evaluate the signature on a file that contains +hashes of the release binaries. The downloaded binaries are then hashed and compared to +the signed checksum file. + +First, you have to figure out which public keys to recognize. Browse the [list of frequent +builder-keys](https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys) and +decide which of these keys you would like to trust. For each key you want to trust, you +must obtain that key for your local GPG installation. + +You can obtain these keys by + - through a browser using a key server (e.g. keyserver.ubuntu.com), + - manually using the `gpg --keyserver <url> --recv-keys <key>` command, or + - you can run the packaged `verify.py ... --import-keys` script to + have it automatically retrieve unrecognized keys. + +#### Usage + +This script attempts to download the checksum file (`SHA256SUMS`) and corresponding +signature file `SHA256SUMS.asc` from https://bitcoincore.org and https://bitcoin.org. + +It first checks if the checksum file is valid based upon a plurality of signatures, and +then downloads the release files specified in the checksum file, and checks if the +hashes of the release files are as expected. + +If we encounter pubkeys in the signature file that we do not recognize, the script +can prompt the user as to whether they'd like to download the pubkeys. To enable +this behavior, use the `--import-keys` flag. + +The script returns 0 if everything passes the checks. It returns 1 if either the +signature check or the hash check doesn't pass. An exit code of >2 indicates an error. + +See the `Config` object for various options. + +#### Examples + +Validate releases with default settings: +```sh +./contrib/verify-binaries/verify.py pub 22.0 +./contrib/verify-binaries/verify.py pub 22.0-rc2 +``` + +Get JSON output and don't prompt for user input (no auto key import): + +```sh +./contrib/verify-binaries/verify.py --json pub 22.0-x86 +``` + +Rely only on local GPG state and manually specified keys, while requiring a +threshold of at least 10 trusted signatures: +```sh +./contrib/verify-binaries/verify.py \ + --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ + --min-good-sigs 10 pub 22.0-x86 +``` + +If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: + +```sh +./contrib/verify-binaries/verify.py pub 22.0-osx +./contrib/verify-binaries/verify.py pub 22.0-rc2-win64 +``` + +If you do not want to keep the downloaded binaries, specify the cleanup option. + +```sh +./contrib/verify-binaries/verify.py pub --cleanup 22.0 +``` + +Use the bin subcommand to verify all files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin SHA256SUMS +``` + +Verify only a subset of the files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin ~/Downloads/SHA256SUMS \ + ~/Downloads/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz \ + ~/Downloads/bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz +``` diff --git a/contrib/verify-binaries/test.py b/contrib/verify-binaries/test.py new file mode 100755 index 0000000000..b7988b86ef --- /dev/null +++ b/contrib/verify-binaries/test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +from pathlib import Path + + +def main(): + """Tests ordered roughly from faster to slower.""" + expect_code(run_verify("", "pub", '0.32'), 4, "Nonexistent version should fail") + expect_code(run_verify("", "pub", '0.32.awefa.12f9h'), 11, "Malformed version should fail") + expect_code(run_verify('--min-good-sigs 20', "pub", "22.0"), 9, "--min-good-sigs 20 should fail") + + print("- testing multisig verification (22.0)", flush=True) + _220 = run_verify("--json", "pub", "22.0") + try: + result = json.loads(_220.stdout.decode()) + except Exception: + print("failed on 22.0 --json:") + print_process_failure(_220) + raise + + expect_code(_220, 0, "22.0 should succeed") + v = result['verified_binaries'] + assert result['good_trusted_sigs'] + assert v['bitcoin-22.0-aarch64-linux-gnu.tar.gz'] == 'ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e' + assert v['bitcoin-22.0-osx64.tar.gz'] == '2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9' + assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' + + +def run_verify(global_args: str, command: str, command_args: str) -> subprocess.CompletedProcess: + maybe_here = Path.cwd() / 'verify.py' + path = maybe_here if maybe_here.exists() else Path.cwd() / 'contrib' / 'verify-binaries' / 'verify.py' + + if command == "pub": + command += " --cleanup" + + return subprocess.run( + f"{path} {global_args} {command} {command_args}", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + +def expect_code(completed: subprocess.CompletedProcess, expected_code: int, msg: str): + if completed.returncode != expected_code: + print(f"{msg!r} failed: got code {completed.returncode}, expected {expected_code}") + print_process_failure(completed) + sys.exit(1) + else: + print(f"✓ {msg!r} passed") + + +def print_process_failure(completed: subprocess.CompletedProcess): + print(f"stdout:\n{completed.stdout.decode()}") + print(f"stderr:\n{completed.stderr.decode()}") + + +if __name__ == '__main__': + main() diff --git a/contrib/verify-binaries/verify.py b/contrib/verify-binaries/verify.py new file mode 100755 index 0000000000..aeab8ebacb --- /dev/null +++ b/contrib/verify-binaries/verify.py @@ -0,0 +1,713 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitcoin Core release binaries. + +This script attempts to download the sum file SHA256SUMS and corresponding +signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and +compares them. + +The sum-signature file is signed by a number of builder keys. This script +ensures that there is a minimum threshold of signatures from pubkeys that +we trust. This trust is articulated on the basis of configuration options +here, but by default is based upon local GPG trust settings. + +The builder keys are available in the guix.sigs repo: + + https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys + +If a minimum good, trusted signature threshold is met on the sum file, we then +download the files specified in SHA256SUMS, and check if the hashes of these +files match those that are specified. The script returns 0 if everything passes +the checks. It returns 1 if either the signature check or the hash check +doesn't pass. If an error occurs the return value is >= 2. + +Logging output goes to stderr and final binary verification data goes to stdout. + +JSON output can by obtained by setting env BINVERIFY_JSON=1. +""" +import argparse +import difflib +import json +import logging +import os +import subprocess +import typing as t +import re +import sys +import shutil +import tempfile +import textwrap +import urllib.request +import urllib.error +import enum +from hashlib import sha256 +from pathlib import PurePath, Path + +# The primary host; this will fail if we can't retrieve files from here. +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SUMS_FILENAME = 'SHA256SUMS' +SIGNATUREFILENAME = f"{SUMS_FILENAME}.asc" + + +class ReturnCode(enum.IntEnum): + SUCCESS = 0 + INTEGRITY_FAILURE = 1 + FILE_GET_FAILED = 4 + FILE_MISSING_FROM_ONE_HOST = 5 + FILES_NOT_EQUAL = 6 + NO_BINARIES_MATCH = 7 + NOT_ENOUGH_GOOD_SIGS = 9 + BINARY_DOWNLOAD_FAILED = 10 + BAD_VERSION = 11 + + +def set_up_logger(is_verbose: bool = True) -> logging.Logger: + """Set up a logger that writes to stderr.""" + log = logging.getLogger(__name__) + log.setLevel(logging.INFO if is_verbose else logging.WARNING) + console = logging.StreamHandler(sys.stderr) # log to stderr + console.setLevel(logging.DEBUG) + formatter = logging.Formatter('[%(levelname)s] %(message)s') + console.setFormatter(formatter) + log.addHandler(console) + return log + + +log = set_up_logger() + + +def indent(output: str) -> str: + return textwrap.indent(output, ' ') + + +def bool_from_env(key, default=False) -> bool: + if key not in os.environ: + return default + raw = os.environ[key] + + if raw.lower() in ('1', 'true'): + return True + elif raw.lower() in ('0', 'false'): + return False + raise ValueError(f"Unrecognized environment value {key}={raw!r}") + + +VERSION_FORMAT = "<major>.<minor>[.<patch>][-rc[0-9]][-platform]" +VERSION_EXAMPLE = "22.0-x86_64 or 0.21.0-rc2-osx" + +def parse_version_string(version_str): + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "<version>-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "<version>-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file): + result = subprocess.run(['wget', '-O', local_file, remote_file], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: + """Get (success, text lines of a file) over HTTP.""" + try: + return (True, [ + line.strip().decode() for line in urllib.request.urlopen(url).readlines()]) + except urllib.error.HTTPError as e: + log.warning(f"HTTP request to {url} failed (HTTPError): {e}") + except Exception as e: + log.warning(f"HTTP request to {url} failed ({e})") + return (False, []) + + +def verify_with_gpg( + filename, + signature_filename, + output_filename: t.Optional[str] = None +) -> t.Tuple[int, str]: + with tempfile.NamedTemporaryFile() as status_file: + args = [ + 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', "--status-file", status_file.name, + '--output', output_filename if output_filename else '', signature_filename, filename] + + env = dict(os.environ, LANGUAGE='en') + result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) + + gpg_data = status_file.read().decode().rstrip() + + log.debug(f'Result from GPG ({result.returncode}): {result.stdout.decode()}') + log.debug(f"{gpg_data}") + return result.returncode, gpg_data + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +class SigData: + """GPG signature data as parsed from GPG stdout.""" + def __init__(self): + self.key = None + self.name = "" + self.trusted = False + self.status = "" + + def __bool__(self): + return self.key is not None + + def __repr__(self): + return ( + "SigData(%r, %r, trusted=%s, status=%r)" % + (self.key, self.name, self.trusted, self.status)) + + +def parse_gpg_result( + output: t.List[str] +) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: + """Returns good, unknown, and bad signatures from GPG stdout.""" + good_sigs: t.List[SigData] = [] + unknown_sigs: t.List[SigData] = [] + bad_sigs: t.List[SigData] = [] + total_resolved_sigs = 0 + + # Ensure that all lines we match on include a prefix that prevents malicious input + # from fooling the parser. + def line_begins_with(patt: str, line: str) -> t.Optional[re.Match]: + return re.match(r'^(\[GNUPG:\])\s+' + patt, line) + + curr_sigs = unknown_sigs + curr_sigdata = SigData() + + for line in output: + if line_begins_with(r"NEWSIG(?:\s|$)", line): + total_resolved_sigs += 1 + if curr_sigdata: + curr_sigs.append(curr_sigdata) + curr_sigdata = SigData() + newsig_split = line.split() + if len(newsig_split) == 3: + curr_sigdata.name = newsig_split[2] + + elif line_begins_with(r"GOODSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + + elif line_begins_with(r"EXPKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "expired" + + elif line_begins_with(r"REVKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "revoked" + + elif line_begins_with(r"BADSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = bad_sigs + + elif line_begins_with(r"ERRSIG(?:\s|$)", line): + curr_sigdata.key, _, _, _, _, _ = line.split()[2:8] + curr_sigs = unknown_sigs + + elif line_begins_with(r"TRUST_(UNDEFINED|NEVER)(?:\s|$)", line): + curr_sigdata.trusted = False + + elif line_begins_with(r"TRUST_(MARGINAL|FULLY|ULTIMATE)(?:\s|$)", line): + curr_sigdata.trusted = True + + # The last one won't have been added, so add it now + assert curr_sigdata + curr_sigs.append(curr_sigdata) + + all_found = len(good_sigs + bad_sigs + unknown_sigs) + if all_found != total_resolved_sigs: + raise RuntimeError( + f"failed to evaluate all signatures: found {all_found} " + f"but expected {total_resolved_sigs}") + + return (good_sigs, unknown_sigs, bad_sigs) + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + eq = contents1 == contents2 + + if not eq: + with open(filename1, 'r', encoding='utf-8') as f1, \ + open(filename2, 'r', encoding='utf-8') as f2: + f1lines = f1.readlines() + f2lines = f2.readlines() + + diff = indent( + ''.join(difflib.unified_diff(f1lines, f2lines))) + log.warning(f"found diff in files ({filename1}, {filename2}):\n{diff}\n") + + return eq + + +def get_files_from_hosts_and_compare( + hosts: t.List[str], path: str, filename: str, require_all: bool = False +) -> ReturnCode: + """ + Retrieve the same file from a number of hosts and ensure they have the same contents. + The first host given will be treated as the "primary" host, and is required to succeed. + + Args: + filename: for writing the file locally. + """ + assert len(hosts) > 1 + primary_host = hosts[0] + other_hosts = hosts[1:] + got_files = [] + + def join_url(host: str) -> str: + return host.rstrip('/') + '/' + path.lstrip('/') + + url = join_url(primary_host) + success, output = download_with_wget(url, filename) + if not success: + log.error( + f"couldn't fetch file ({url}). " + "Have you specified the version number in the following format?\n" + f"{VERSION_FORMAT} " + f"(example: {VERSION_EXAMPLE})\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_GET_FAILED + else: + log.info(f"got file {url} as {filename}") + got_files.append(filename) + + for i, host in enumerate(other_hosts): + url = join_url(host) + fname = filename + f'.{i + 2}' + success, output = download_with_wget(url, fname) + + if require_all and not success: + log.error( + f"{host} failed to provide file ({url}), but {primary_host} did?\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_MISSING_FROM_ONE_HOST + elif not success: + log.warning( + f"{host} failed to provide file ({url}). " + f"Continuing based solely upon {primary_host}.") + else: + log.info(f"got file {url} as {fname}") + got_files.append(fname) + + for i, got_file in enumerate(got_files): + if got_file == got_files[-1]: + break # break on last file, nothing after it to compare to + + compare_to = got_files[i + 1] + if not files_are_equal(got_file, compare_to): + log.error(f"files not equal: {got_file} and {compare_to}") + return ReturnCode.FILES_NOT_EQUAL + + return ReturnCode.SUCCESS + + +def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: + # check signature + # + # We don't write output to a file because this command will almost certainly + # fail with GPG exit code '2' (and so not writing to --output) because of the + # likely presence of multiple untrusted signatures. + retval, output = verify_with_gpg(sums_file, sigfilename) + + if args.verbose: + log.info(f"gpg output:\n{indent(output)}") + + good, unknown, bad = parse_gpg_result(output.splitlines()) + + if unknown and args.import_keys: + # Retrieve unknown keys and then try GPG again. + for unsig in unknown: + if prompt_yn(f" ? Retrieve key {unsig.key} ({unsig.name})? (y/N) "): + ran = subprocess.run( + ["gpg", "--keyserver", args.keyserver, "--recv-keys", unsig.key]) + + if ran.returncode != 0: + log.warning(f"failed to retrieve key {unsig.key}") + + # Reparse the GPG output now that we have more keys + retval, output = verify_with_gpg(sums_file, sigfilename) + good, unknown, bad = parse_gpg_result(output.splitlines()) + + return retval, output, good, unknown, bad + + +def prompt_yn(prompt) -> bool: + """Return true if the user inputs 'y'.""" + got = '' + while got not in ['y', 'n']: + got = input(prompt).lower() + return got == 'y' + +def verify_shasums_signature( + signature_file_path: str, sums_file_path: str, args: argparse.Namespace +) -> t.Tuple[ + ReturnCode, t.List[SigData], t.List[SigData], t.List[SigData], t.List[SigData] +]: + min_good_sigs = args.min_good_sigs + gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. + + gpg_retval, gpg_output, good, unknown, bad = check_multisig(sums_file_path, signature_file_path, args) + + if gpg_retval not in gpg_allowed_codes: + if gpg_retval == 1: + log.critical(f"Bad signature (code: {gpg_retval}).") + else: + log.critical(f"unexpected GPG exit code ({gpg_retval})") + + log.error(f"gpg output:\n{indent(gpg_output)}") + return (ReturnCode.INTEGRITY_FAILURE, [], [], [], []) + + # Decide which keys we trust, though not "trust" in the GPG sense, but rather + # which pubkeys convince us that this sums file is legitimate. In other words, + # which pubkeys within the Bitcoin community do we trust for the purposes of + # binary verification? + trusted_keys = set() + if args.trusted_keys: + trusted_keys |= set(args.trusted_keys.split(',')) + + # Tally signatures and make sure we have enough goods to fulfill + # our threshold. + good_trusted = [sig for sig in good if sig.trusted or sig.key in trusted_keys] + good_untrusted = [sig for sig in good if sig not in good_trusted] + num_trusted = len(good_trusted) + len(good_untrusted) + log.info(f"got {num_trusted} good signatures") + + if num_trusted < min_good_sigs: + log.info("Maybe you need to import " + f"(`gpg --keyserver {args.keyserver} --recv-keys <key-id>`) " + "some of the following keys: ") + log.info('') + for sig in unknown: + log.info(f" {sig.key} ({sig.name})") + log.info('') + log.error( + "not enough trusted sigs to meet threshold " + f"({num_trusted} vs. {min_good_sigs})") + + return (ReturnCode.NOT_ENOUGH_GOOD_SIGS, [], [], [], []) + + for sig in good_trusted: + log.info(f"GOOD SIGNATURE: {sig}") + + for sig in good_untrusted: + log.info(f"GOOD SIGNATURE (untrusted): {sig}") + + for sig in [sig for sig in good if sig.status == 'expired']: + log.warning(f"key {sig.key} for {sig.name} is expired") + + for sig in bad: + log.warning(f"BAD SIGNATURE: {sig}") + + for sig in unknown: + log.warning(f"UNKNOWN SIGNATURE: {sig}") + + return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) + + +def parse_sums_file(sums_file_path: str, filename_filter: t.List[str]) -> t.List[t.List[str]]: + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: "<hash> <binary_filename>" + with open(sums_file_path, 'r', encoding='utf8') as hash_file: + return [line.split()[:2] for line in hash_file if len(filename_filter) == 0 or any(f in line for f in filename_filter)] + + +def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: + offending_files = [] + files_to_hashes = {} + + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + else: + files_to_hashes[binary_filename] = hash_calculated + + if offending_files: + joined_files = '\n'.join(offending_files) + log.critical( + "Hashes don't match.\n" + f"Offending files:\n{joined_files}") + return (ReturnCode.INTEGRITY_FAILURE, files_to_hashes) + + return (ReturnCode.SUCCESS, files_to_hashes) + + +def verify_published_handler(args: argparse.Namespace) -> ReturnCode: + WORKINGDIR = Path(tempfile.gettempdir()) / f"bitcoin_verify_binaries.{args.version}" + + def cleanup(): + log.info("cleaning up files") + os.chdir(Path.home()) + shutil.rmtree(WORKINGDIR) + + # determine remote dir dependent on provided version string + try: + version_base, version_rc, os_filter = parse_version_string(args.version) + version_tuple = [int(i) for i in version_base.split('.')] + except Exception as e: + log.debug(e) + log.error(f"unable to parse version; expected format is {VERSION_FORMAT}") + log.error(f" e.g. {VERSION_EXAMPLE}") + return ReturnCode.BAD_VERSION + + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigs_path = remote_dir + SIGNATUREFILENAME + remote_sums_path = remote_dir + SUMS_FILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + hosts = [HOST1, HOST2] + + got_sig_status = get_files_from_hosts_and_compare( + hosts, remote_sigs_path, SIGNATUREFILENAME, args.require_all_hosts) + if got_sig_status != ReturnCode.SUCCESS: + return got_sig_status + + # Multi-sig verification is available after 22.0. + if version_tuple[0] < 22: + log.error("Version too old - single sig not supported. Use a previous " + "version of this script from the repo.") + return ReturnCode.BAD_VERSION + + got_sums_status = get_files_from_hosts_and_compare( + hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) + if got_sums_status != ReturnCode.SUCCESS: + return got_sums_status + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(SIGNATUREFILENAME, SUMS_FILENAME, args) + if sigs_status != ReturnCode.SUCCESS: + if sigs_status == ReturnCode.INTEGRITY_FAILURE: + cleanup() + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(SUMS_FILENAME, [os_filter]) + if not hashes_to_verify: + log.error("no files matched the platform specified") + return ReturnCode.NO_BINARIES_MATCH + + # remove binaries that are known not to be hosted by bitcoincore.org + fragments_to_remove = ['-unsigned', '-debug', '-codesignatures'] + for fragment in fragments_to_remove: + nobinaries = [i for i in hashes_to_verify if fragment in i[1]] + if nobinaries: + remove_str = ', '.join(i[1] for i in nobinaries) + log.info( + f"removing *{fragment} binaries ({remove_str}) from verification " + f"since {HOST1} does not host *{fragment} binaries") + hashes_to_verify = [i for i in hashes_to_verify if fragment not in i[1]] + + # download binaries + for _, binary_filename in hashes_to_verify: + log.info(f"downloading {binary_filename}") + success, output = download_with_wget( + HOST1 + remote_dir + binary_filename, binary_filename) + + if not success: + log.error( + f"failed to download {binary_filename}\n" + f"wget output:\n{indent(output)}") + return ReturnCode.BINARY_DOWNLOAD_FAILED + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(hashes_to_verify) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + + if args.cleanup: + cleanup() + else: + log.info(f"did not clean up {WORKINGDIR}") + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + + return ReturnCode.SUCCESS + + +def verify_binaries_handler(args: argparse.Namespace) -> ReturnCode: + binary_to_basename = {} + for file in args.binary: + binary_to_basename[PurePath(file).name] = file + + sums_sig_path = None + if args.sums_sig_file: + sums_sig_path = Path(args.sums_sig_file) + else: + log.info(f"No signature file specified, assuming it is {args.sums_file}.asc") + sums_sig_path = Path(args.sums_file).with_suffix(".asc") + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(str(sums_sig_path), args.sums_file, args) + if sigs_status != ReturnCode.SUCCESS: + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(args.sums_file, [k for k, n in binary_to_basename.items()]) + if not hashes_to_verify: + log.error(f"No files in {args.sums_file} match the specified binaries") + return ReturnCode.NO_BINARIES_MATCH + + # Make sure all files are accounted for + sums_file_path = Path(args.sums_file) + missing_files = [] + files_to_hash = [] + if len(binary_to_basename) > 0: + for file_hash, file in hashes_to_verify: + files_to_hash.append([file_hash, binary_to_basename[file]]) + del binary_to_basename[file] + if len(binary_to_basename) > 0: + log.error(f"Not all specified binaries are in {args.sums_file}") + return ReturnCode.NO_BINARIES_MATCH + else: + log.info(f"No binaries specified, assuming all files specified in {args.sums_file} are located relatively") + for file_hash, file in hashes_to_verify: + file_path = Path(sums_file_path.parent.joinpath(file)) + if file_path.exists(): + files_to_hash.append([file_hash, str(file_path)]) + else: + missing_files.append(file) + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(files_to_hash) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + "missing_binaries": missing_files, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + for filename in missing_files: + print(f"MISSING: {filename}") + + return ReturnCode.SUCCESS + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '-v', '--verbose', action='store_true', + default=bool_from_env('BINVERIFY_VERBOSE'), + ) + parser.add_argument( + '-q', '--quiet', action='store_true', + default=bool_from_env('BINVERIFY_QUIET'), + ) + parser.add_argument( + '--import-keys', action='store_true', + default=bool_from_env('BINVERIFY_IMPORTKEYS'), + help='if specified, ask to import each unknown builder key' + ) + parser.add_argument( + '--min-good-sigs', type=int, action='store', nargs='?', + default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), + help=( + 'The minimum number of good signatures to require successful termination.'), + ) + parser.add_argument( + '--keyserver', action='store', nargs='?', + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkps://keys.openpgp.org'), + help='which keyserver to use', + ) + parser.add_argument( + '--trusted-keys', action='store', nargs='?', + default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), + help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', + ) + parser.add_argument( + '--json', action='store_true', + default=bool_from_env('BINVERIFY_JSON'), + help='If set, output the result as JSON', + ) + + subparsers = parser.add_subparsers(title="Commands", required=True, dest="command") + + pub_parser = subparsers.add_parser("pub", help="Verify a published release.") + pub_parser.set_defaults(func=verify_published_handler) + pub_parser.add_argument( + 'version', type=str, help=( + f'version of the bitcoin release to download; of the format ' + f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') + ) + pub_parser.add_argument( + '--cleanup', action='store_true', + default=bool_from_env('BINVERIFY_CLEANUP'), + help='if specified, clean up files afterwards' + ) + pub_parser.add_argument( + '--require-all-hosts', action='store_true', + default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), + help=( + f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' + '(Sometimes bitcoin.org lags behind bitcoincore.org.)') + ) + + bin_parser = subparsers.add_parser("bin", help="Verify local binaries.") + bin_parser.set_defaults(func=verify_binaries_handler) + bin_parser.add_argument("--sums-sig-file", "-s", help="Path to the SHA256SUMS.asc file to verify") + bin_parser.add_argument("sums_file", help="Path to the SHA256SUMS file to verify") + bin_parser.add_argument( + "binary", nargs="*", + help="Path to a binary distribution file to verify. Can be specified multiple times for multiple files to verify." + ) + + args = parser.parse_args() + if args.quiet: + log.setLevel(logging.WARNING) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) |