aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.travis.yml2
-rw-r--r--contrib/verify-commits/README.md10
-rw-r--r--contrib/verify-commits/allow-incorrect-sha512-commits2
-rw-r--r--contrib/verify-commits/allow-unclean-merge-commits4
-rwxr-xr-xcontrib/verify-commits/pre-push-hook.sh4
-rwxr-xr-xcontrib/verify-commits/verify-commits.py155
-rwxr-xr-xcontrib/verify-commits/verify-commits.sh153
7 files changed, 169 insertions, 161 deletions
diff --git a/.travis.yml b/.travis.yml
index 7cbe0b83f1..42ea5fdc44 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -104,5 +104,5 @@ jobs:
- test/lint/lint-all.sh
- if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then
while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys &&
- travis_wait 30 contrib/verify-commits/verify-commits.sh;
+ travis_wait 30 contrib/verify-commits/verify-commits.py;
fi
diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md
index fa492fdd27..aa805ad1b9 100644
--- a/contrib/verify-commits/README.md
+++ b/contrib/verify-commits/README.md
@@ -7,18 +7,18 @@ are PGP signed (nearly always merge commits), as well as a script to verify
commits against a trusted keys list.
-Using verify-commits.sh safely
+Using verify-commits.py safely
------------------------------
Remember that you can't use an untrusted script to verify itself. This means
-that checking out code, then running `verify-commits.sh` against `HEAD` is
-_not_ safe, because the version of `verify-commits.sh` that you just ran could
+that checking out code, then running `verify-commits.py` against `HEAD` is
+_not_ safe, because the version of `verify-commits.py` that you just ran could
be backdoored. Instead, you need to use a trusted version of verify-commits
prior to checkout to make sure you're checking out only code signed by trusted
keys:
git fetch origin && \
- ./contrib/verify-commits/verify-commits.sh origin/master && \
+ ./contrib/verify-commits/verify-commits.py origin/master && \
git checkout origin/master
Note that the above isn't a good UI/UX yet, and needs significant improvements
@@ -42,6 +42,6 @@ said key. In order to avoid bumping the root-of-trust `trusted-git-root`
file, individual commits which were signed by such a key can be added to the
`allow-revsig-commits` file. That way, the PGP signatures are still verified
but no new commits can be signed by any expired/revoked key. To easily build a
-list of commits which need to be added, verify-commits.sh can be edited to test
+list of commits which need to be added, verify-commits.py can be edited to test
each commit with BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG set to both 1 and 0, and
those which need it set to 1 printed.
diff --git a/contrib/verify-commits/allow-incorrect-sha512-commits b/contrib/verify-commits/allow-incorrect-sha512-commits
new file mode 100644
index 0000000000..c572806f26
--- /dev/null
+++ b/contrib/verify-commits/allow-incorrect-sha512-commits
@@ -0,0 +1,2 @@
+f8feaa4636260b599294c7285bcf1c8b7737f74e
+8040ae6fc576e9504186f2ae3ff2c8125de1095c
diff --git a/contrib/verify-commits/allow-unclean-merge-commits b/contrib/verify-commits/allow-unclean-merge-commits
new file mode 100644
index 0000000000..7aab274b9a
--- /dev/null
+++ b/contrib/verify-commits/allow-unclean-merge-commits
@@ -0,0 +1,4 @@
+6052d509105790a26b3ad5df43dd61e7f1b24a12
+3798e5de334c3deb5f71302b782f6b8fbd5087f1
+326ffed09bfcc209a2efd6a2ebc69edf6bd200b5
+97d83739db0631be5d4ba86af3616014652c00ec
diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh
index c21febb9e9..1f14635f61 100755
--- a/contrib/verify-commits/pre-push-hook.sh
+++ b/contrib/verify-commits/pre-push-hook.sh
@@ -12,9 +12,9 @@ while read LINE; do
if [ "$4" != "refs/heads/master" ]; then
continue
fi
- if ! ./contrib/verify-commits/verify-commits.sh $3 > /dev/null 2>&1; then
+ if ! ./contrib/verify-commits/verify-commits.py $3 > /dev/null 2>&1; then
echo "ERROR: A commit is not signed, can't push"
- ./contrib/verify-commits/verify-commits.sh
+ ./contrib/verify-commits/verify-commits.py
exit 1
fi
done < /dev/stdin
diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py
new file mode 100755
index 0000000000..80f0aa0bf1
--- /dev/null
+++ b/contrib/verify-commits/verify-commits.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Verify commits against a trusted keys list."""
+import argparse
+import hashlib
+import os
+import subprocess
+import sys
+import time
+
+GIT = os.getenv('GIT', 'git')
+
+def tree_sha512sum(commit='HEAD'):
+ """Calculate the Tree-sha512 for the commit.
+
+ This is copied from github-merge.py."""
+
+ # request metadata for entire tree, recursively
+ files = []
+ blob_by_name = {}
+ for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
+ name_sep = line.index(b'\t')
+ metadata = line[:name_sep].split() # perms, 'blob', blobid
+ assert metadata[1] == b'blob'
+ name = line[name_sep + 1:]
+ files.append(name)
+ blob_by_name[name] = metadata[2]
+
+ files.sort()
+ # open connection to git-cat-file in batch mode to request data for all blobs
+ # this is much faster than launching it per file
+ p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+ overall = hashlib.sha512()
+ for f in files:
+ blob = blob_by_name[f]
+ # request blob
+ p.stdin.write(blob + b'\n')
+ p.stdin.flush()
+ # read header: blob, "blob", size
+ reply = p.stdout.readline().split()
+ assert reply[0] == blob and reply[1] == b'blob'
+ size = int(reply[2])
+ # hash the blob data
+ intern = hashlib.sha512()
+ ptr = 0
+ while ptr < size:
+ bs = min(65536, size - ptr)
+ piece = p.stdout.read(bs)
+ if len(piece) == bs:
+ intern.update(piece)
+ else:
+ raise IOError('Premature EOF reading git cat-file output')
+ ptr += bs
+ dig = intern.hexdigest()
+ assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data
+ # update overall hash with file hash
+ overall.update(dig.encode("utf-8"))
+ overall.update(" ".encode("utf-8"))
+ overall.update(f)
+ overall.update("\n".encode("utf-8"))
+ p.stdin.close()
+ if p.wait():
+ raise IOError('Non-zero return value executing git cat-file')
+ return overall.hexdigest()
+
+def main():
+ # Parse arguments
+ parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
+ parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
+ parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
+ parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
+ args = parser.parse_args()
+
+ # get directory of this program and read data files
+ dirname = os.path.dirname(os.path.abspath(__file__))
+ print("Using verify-commits data from " + dirname)
+ verified_root = open(dirname + "/trusted-git-root", "r").read().splitlines()[0]
+ verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r").read().splitlines()[0]
+ revsig_allowed = open(dirname + "/allow-revsig-commits", "r").read().splitlines()
+ unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r").read().splitlines()
+ incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r").read().splitlines()
+
+ # Set commit and branch and set variables
+ current_commit = args.commit
+ if ' ' in current_commit:
+ print("Commit must not contain spaces", file=sys.stderr)
+ sys.exit(1)
+ verify_tree = args.verify_tree
+ no_sha1 = True
+ prev_commit = ""
+ initial_commit = current_commit
+ branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True).splitlines()[0]
+
+ # Iterate through commits
+ while True:
+ if current_commit == verified_root:
+ print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
+ sys.exit(0)
+ if current_commit == verified_sha512_root:
+ if verify_tree:
+ print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
+ verify_tree = False
+ no_sha1 = False
+
+ os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
+ os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0"
+
+ # Check that the commit (and parents) was signed with a trusted key
+ if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL):
+ if prev_commit != "":
+ print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
+ print("Parents are:", file=sys.stderr)
+ parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True).splitlines()[0].split(' ')
+ for parent in parents:
+ subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
+ else:
+ print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
+ sys.exit(1)
+
+ # Check the Tree-SHA512
+ if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
+ tree_hash = tree_sha512sum(current_commit)
+ if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True).splitlines():
+ print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
+ sys.exit(1)
+
+ # Merge commits should only have two parents
+ parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True).splitlines()[0].split(' ')
+ if len(parents) > 2:
+ print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
+ sys.exit(1)
+
+ # Check that the merge commit is clean
+ commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True).splitlines()[0])
+ check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
+ allow_unclean = current_commit in unclean_merge_allowed
+ if len(parents) == 2 and check_merge and not allow_unclean:
+ current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True).splitlines()[0]
+ subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
+ subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL)
+ recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True).splitlines()[0]
+ if current_tree != recreated_tree:
+ print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
+ subprocess.call([GIT, 'diff', current_commit])
+ subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
+ sys.exit(1)
+ subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
+
+ prev_commit = current_commit
+ current_commit = parents[0]
+
+if __name__ == '__main__':
+ main()
diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh
deleted file mode 100755
index 6415eea4d5..0000000000
--- a/contrib/verify-commits/verify-commits.sh
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2014-2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-DIR=$(dirname "$0")
-[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0")
-
-echo "Using verify-commits data from ${DIR}"
-
-VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root")
-VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit")
-REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits")
-
-HAVE_GNU_SHA512=1
-[ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0
-
-if [ x"$1" = "x" ]; then
- CURRENT_COMMIT="HEAD"
-else
- CURRENT_COMMIT="$1"
-fi
-
-if [ "${CURRENT_COMMIT#* }" != "$CURRENT_COMMIT" ]; then
- echo "Commit must not contain spaces?" > /dev/stderr
- exit 1
-fi
-
-VERIFY_TREE=0
-if [ x"$2" = "x--tree-checks" ]; then
- VERIFY_TREE=1
-fi
-
-NO_SHA1=1
-PREV_COMMIT=""
-INITIAL_COMMIT="${CURRENT_COMMIT}"
-
-BRANCH="$(git rev-parse --abbrev-ref HEAD)"
-
-while true; do
- if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then
- echo "There is a valid path from \"$INITIAL_COMMIT\" to $VERIFIED_ROOT where all commits are signed!"
- exit 0
- fi
-
- if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then
- if [ "$VERIFY_TREE" = "1" ]; then
- echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr
- fi
- VERIFY_TREE=0
- NO_SHA1=0
- fi
-
- if [ "$NO_SHA1" = "1" ]; then
- export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0
- else
- export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1
- fi
-
- if [ "${REVSIG_ALLOWED#*$CURRENT_COMMIT}" != "$REVSIG_ALLOWED" ]; then
- export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1
- else
- export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0
- fi
-
- if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit "$CURRENT_COMMIT" > /dev/null; then
- if [ "$PREV_COMMIT" != "" ]; then
- echo "No parent of $PREV_COMMIT was signed with a trusted key!" > /dev/stderr
- echo "Parents are:" > /dev/stderr
- PARENTS=$(git show -s --format=format:%P $PREV_COMMIT)
- for PARENT in $PARENTS; do
- git show -s $PARENT > /dev/stderr
- done
- else
- echo "$CURRENT_COMMIT was not signed with a trusted key!" > /dev/stderr
- fi
- exit 1
- fi
-
- # We always verify the top of the tree
- if [ "$VERIFY_TREE" = 1 -o "$PREV_COMMIT" = "" ]; then
- IFS_CACHE="$IFS"
- IFS='
-'
- for LINE in $(git ls-tree --full-tree -r "$CURRENT_COMMIT"); do
- case "$LINE" in
- "12"*)
- echo "Repo contains symlinks" > /dev/stderr
- IFS="$IFS_CACHE"
- exit 1
- ;;
- esac
- done
- IFS="$IFS_CACHE"
-
- FILE_HASHES=""
- for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do
- if [ "$HAVE_GNU_SHA512" = 1 ]; then
- HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST _; echo $FIRST; } )
- else
- HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST _; echo $FIRST; } )
- fi
- [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"'
-'
- FILE_HASHES="$FILE_HASHES$HASH $FILE"
- done
-
- if [ "$HAVE_GNU_SHA512" = 1 ]; then
- TREE_HASH="$(echo "$FILE_HASHES" | sha512sum)"
- else
- TREE_HASH="$(echo "$FILE_HASHES" | shasum -a 512)"
- fi
- HASH_MATCHES=0
- MSG="$(git show -s --format=format:%B "$CURRENT_COMMIT" | tail -n1)"
-
- case "$MSG -" in
- "Tree-SHA512: $TREE_HASH")
- HASH_MATCHES=1;;
- esac
-
- if [ "$HASH_MATCHES" = "0" ]; then
- echo "Tree-SHA512 did not match for commit $CURRENT_COMMIT" > /dev/stderr
- exit 1
- fi
- fi
-
- PARENTS=$(git show -s --format=format:%P "$CURRENT_COMMIT")
- PARENT1=${PARENTS%% *}
- PARENT2=""
- if [ "x$PARENT1" != "x$PARENTS" ]; then
- PARENTX=${PARENTS#* }
- PARENT2=${PARENTX%% *}
- if [ "x$PARENT2" != "x$PARENTX" ]; then
- echo "Commit $CURRENT_COMMIT is an octopus merge" > /dev/stderr
- exit 1
- fi
- fi
- if [ "x$PARENT2" != "x" ]; then
- CURRENT_TREE="$(git show --format="%T" "$CURRENT_COMMIT")"
- git checkout --force --quiet "$PARENT1"
- git merge --no-ff --quiet "$PARENT2" >/dev/null
- RECREATED_TREE="$(git show --format="%T" HEAD)"
- if [ "$CURRENT_TREE" != "$RECREATED_TREE" ]; then
- echo "Merge commit $CURRENT_COMMIT is not clean" > /dev/stderr
- git diff "$CURRENT_COMMIT"
- git checkout --force --quiet "$BRANCH"
- exit 1
- fi
- git checkout --force --quiet "$BRANCH"
- fi
- PREV_COMMIT="$CURRENT_COMMIT"
- CURRENT_COMMIT="$PARENT1"
-done