diff options
Diffstat (limited to 'contrib')
58 files changed, 1537 insertions, 1774 deletions
diff --git a/contrib/debian/bitcoin-qt.desktop b/contrib/debian/bitcoin-qt.desktop index 204cdf99d0..8b31222648 100644 --- a/contrib/debian/bitcoin-qt.desktop +++ b/contrib/debian/bitcoin-qt.desktop @@ -10,5 +10,5 @@ Terminal=false Type=Application Icon=bitcoin128 MimeType=x-scheme-handler/bitcoin; -Categories=Office;Finance; +Categories=Office;Finance;P2P;Network;Qt; StartupWMClass=Bitcoin-qt diff --git a/contrib/debian/copyright b/contrib/debian/copyright index c6484157a5..21cca7d9ac 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -76,8 +76,8 @@ Comment: Files: src/qt/res/icons/clock*.png src/qt/res/icons/eye_*.png - src/qt/res/icons/verify.png src/qt/res/icons/tx_in*.png + src/qt/res/icons/verify.png src/qt/res/src/clock_*.svg src/qt/res/src/tx_*.svg src/qt/res/src/verify.svg @@ -93,6 +93,11 @@ Copyright: Bitboy, Jonas Schnelli License: public-domain Comment: Site: https://bitcointalk.org/?topic=1756.0 +Files: src/qt/res/icons/proxy.png + src/qt/res/src/proxy.svg +Copyright: Cristian Mircea Messel +Licese: public-domain + License: Expat Permission is hereby granted, free of charge, to any person obtaining a diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 8ca8fa9066..a0b6225345 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -2,12 +2,6 @@ Contents ======== This directory contains tools for developers working on this repository. -check-doc.py -============ - -Check if all command line args are documented. The return value indicates the -number of undocumented args. - clang-format-diff.py =================== @@ -93,23 +87,6 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh ``` -git-subtree-check.sh -==================== - -Run this script from the root of the repository to verify that a subtree matches the contents of -the commit it claims to have been updated to. - -To use, make sure that you have fetched the upstream repository branch in which the subtree is -maintained: -* for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) -* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) -* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) -* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) - -Usage: `git-subtree-check.sh DIR (COMMIT)` - -`COMMIT` may be omitted, in which case `HEAD` is used. - github-merge.py =============== @@ -144,6 +121,14 @@ Configuring the github-merge tool for the bitcoin repository is done in the foll git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) git config --global user.signingkey mykeyid (if you want to GPG sign) +Create and verify timestamps of merge commits +--------------------------------------------- +To create or verify timestamps on the merge commits, install the OpenTimestamps +client via `pip3 install opentimestamps-client`. Then, dowload the gpg wrapper +`ots-git-gpg-wrapper.sh` and set it as git's `gpg.program`. See +[the ots git integration documentation](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md#usage) +for further details. + optimize-pngs.py ================ @@ -186,3 +171,14 @@ It will do the following automatically: - add missing translations to the build system (TODO) See doc/translation-process.md for more information. + +circular-dependencies.py +======================== + +Run this script from the root of the source tree (`src/`) to find circular dependencies in the source code. +This looks only at which files include other files, treating the `.cpp` and `.h` file as one unit. + +Example usage: + + cd .../src + ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py deleted file mode 100755 index f164ea9322..0000000000 --- a/contrib/devtools/check-doc.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2015-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -''' -This checks if all command line args are documented. -Return value is 0 to indicate no error. - -Author: @MarcoFalke -''' - -from subprocess import check_output -import re -import sys - -FOLDER_GREP = 'src' -FOLDER_TEST = 'src/test/' -CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/%s' % FOLDER_GREP -CMD_GREP_ARGS = r"egrep -r -I '(map(Multi)?Args(\.count\(|\[)|Get(Bool)?Arg\()\"\-[^\"]+?\"' %s | grep -v '%s'" % (CMD_ROOT_DIR, FOLDER_TEST) -CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' %s" % (CMD_ROOT_DIR) -REGEX_ARG = re.compile(r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"') -REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")') -# list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-dbcrashratio', '-forcecompactdb', '-usehd']) - -def main(): - used = check_output(CMD_GREP_ARGS, shell=True) - docd = check_output(CMD_GREP_DOCS, shell=True) - - args_used = set(re.findall(REGEX_ARG,used)) - args_docd = set(re.findall(REGEX_DOC,docd)).union(SET_DOC_OPTIONAL) - args_need_doc = args_used.difference(args_docd) - args_unknown = args_docd.difference(args_used) - - print "Args used : %s" % len(args_used) - print "Args documented : %s" % len(args_docd) - print "Args undocumented: %s" % len(args_need_doc) - print args_need_doc - print "Args unknown : %s" % len(args_unknown) - print args_unknown - - sys.exit(len(args_need_doc)) - -if __name__ == "__main__": - main() diff --git a/contrib/devtools/check-rpc-mappings.py b/contrib/devtools/check-rpc-mappings.py deleted file mode 100755 index 7e96852c5c..0000000000 --- a/contrib/devtools/check-rpc-mappings.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Check RPC argument consistency.""" - -from collections import defaultdict -import os -import re -import sys - -# Source files (relative to root) to scan for dispatch tables -SOURCES = [ - "src/rpc/server.cpp", - "src/rpc/blockchain.cpp", - "src/rpc/mining.cpp", - "src/rpc/misc.cpp", - "src/rpc/net.cpp", - "src/rpc/rawtransaction.cpp", - "src/wallet/rpcwallet.cpp", -] -# Source file (relative to root) containing conversion mapping -SOURCE_CLIENT = 'src/rpc/client.cpp' -# Argument names that should be ignored in consistency checks -IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'} - -class RPCCommand: - def __init__(self, name, args): - self.name = name - self.args = args - -class RPCArgument: - def __init__(self, names, idx): - self.names = names - self.idx = idx - self.convert = False - -def parse_string(s): - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - -def process_commands(fname): - """Find and parse dispatch table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if re.match("static const CRPCCommand .*\[\] =", line): - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search('{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(2)) - args_str = m.group(4).strip() - if args_str: - args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))] - else: - args = [] - cmds.append(RPCCommand(name, args)) - assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps" - return cmds - -def process_mapping(fname): - """Find and parse conversion table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if line == 'static const CRPCConvertParam vRPCConvertParams[] =': - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search('{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(1)) - idx = int(m.group(2)) - argname = parse_string(m.group(3)) - cmds.append((name, idx, argname)) - assert not in_rpcs and cmds - return cmds - -def main(): - root = sys.argv[1] - - # Get all commands from dispatch tables - cmds = [] - for fname in SOURCES: - cmds += process_commands(os.path.join(root, fname)) - - cmds_by_name = {} - for cmd in cmds: - cmds_by_name[cmd.name] = cmd - - # Get current convert mapping for client - client = SOURCE_CLIENT - mapping = set(process_mapping(os.path.join(root, client))) - - print('* Checking consistency between dispatch tables and vRPCConvertParams') - - # Check mapping consistency - errors = 0 - for (cmdname, argidx, argname) in mapping: - try: - rargnames = cmds_by_name[cmdname].args[argidx].names - except IndexError: - print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname)) - errors += 1 - continue - if argname not in rargnames: - print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr) - errors += 1 - - # Check for conflicts in vRPCConvertParams conversion - # All aliases for an argument must either be present in the - # conversion table, or not. Anything in between means an oversight - # and some aliases won't work. - for cmd in cmds: - for arg in cmd.args: - convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names] - if any(convert) != all(convert): - print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert)) - errors += 1 - arg.convert = all(convert) - - # Check for conversion difference by argument name. - # It is preferable for API consistency that arguments with the same name - # have the same conversion, so bin by argument name. - all_methods_by_argname = defaultdict(list) - converts_by_argname = defaultdict(list) - for cmd in cmds: - for arg in cmd.args: - for argname in arg.names: - all_methods_by_argname[argname].append(cmd.name) - converts_by_argname[argname].append(arg.convert) - - for argname, convert in converts_by_argname.items(): - if all(convert) != any(convert): - if argname in IGNORE_DUMMY_ARGS: - # these are testing or dummy, don't warn for them - continue - print('WARNING: conversion mismatch for argument named %s (%s)' % - (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) - - sys.exit(errors > 0) - - -if __name__ == '__main__': - main() diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py new file mode 100755 index 0000000000..abfa5ed5ae --- /dev/null +++ b/contrib/devtools/circular-dependencies.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import sys +import re + +MAPPING = { + 'core_read.cpp': 'core_io.cpp', + 'core_write.cpp': 'core_io.cpp', +} + +def module_name(path): + if path in MAPPING: + path = MAPPING[path] + if path.endswith(".h"): + return path[:-2] + if path.endswith(".c"): + return path[:-2] + if path.endswith(".cpp"): + return path[:-4] + return None + +files = dict() +deps = dict() + +RE = re.compile("^#include <(.*)>") + +# Iterate over files, and create list of modules +for arg in sys.argv[1:]: + module = module_name(arg) + if module is None: + print("Ignoring file %s (does not constitute module)\n" % arg) + else: + files[arg] = module + deps[module] = set() + +# Iterate again, and build list of direct dependencies for each module +# TODO: implement support for multiple include directories +for arg in sorted(files.keys()): + module = files[arg] + with open(arg, 'r', encoding="utf8") as f: + for line in f: + match = RE.match(line) + if match: + include = match.group(1) + included_module = module_name(include) + if included_module is not None and included_module in deps and included_module != module: + deps[module].add(included_module) + +# Loop to find the shortest (remaining) circular dependency +have_cycle = False +while True: + shortest_cycle = None + for module in sorted(deps.keys()): + # Build the transitive closure of dependencies of module + closure = dict() + for dep in deps[module]: + closure[dep] = [] + while True: + old_size = len(closure) + old_closure_keys = sorted(closure.keys()) + for src in old_closure_keys: + for dep in deps[src]: + if dep not in closure: + closure[dep] = closure[src] + [src] + if len(closure) == old_size: + break + # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest + if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)): + shortest_cycle = [module] + closure[module] + if shortest_cycle is None: + break + # We have the shortest circular dependency; report it + module = shortest_cycle[0] + print("Circular dependency: %s" % (" -> ".join(shortest_cycle + [module]))) + # And then break the dependency to avoid repeating in other cycles + deps[shortest_cycle[-1]] = deps[shortest_cycle[-1]] - set([module]) + have_cycle = True + +sys.exit(1 if have_cycle else 0) diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 7ea49b65e1..77e845a9b4 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # #===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# # @@ -69,10 +69,9 @@ Example usage for git/svn users: import argparse import difflib +import io import re -import string import subprocess -import StringIO import sys @@ -133,9 +132,9 @@ def main(): ['-lines', str(start_line) + ':' + str(end_line)]) # Reformat files containing changes in place. - for filename, lines in lines_by_file.iteritems(): + for filename, lines in lines_by_file.items(): if args.i and args.verbose: - print 'Formatting', filename + print('Formatting {}'.format(filename)) command = [binary, filename] if args.i: command.append('-i') @@ -143,20 +142,23 @@ def main(): command.append('-sort-includes') command.extend(lines) command.extend(['-style=file', '-fallback-style=none']) - p = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=None, stdin=subprocess.PIPE) + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True) stdout, stderr = p.communicate() if p.returncode != 0: sys.exit(p.returncode) if not args.i: - with open(filename) as f: + with open(filename, encoding="utf8") as f: code = f.readlines() - formatted_code = StringIO.StringIO(stdout).readlines() + formatted_code = io.StringIO(stdout).readlines() diff = difflib.unified_diff(code, formatted_code, filename, filename, '(before formatting)', '(after formatting)') - diff_string = string.join(diff, '') + diff_string = ''.join(diff) if len(diff_string) > 0: sys.stdout.write(diff_string) diff --git a/contrib/devtools/commit-script-check.sh b/contrib/devtools/commit-script-check.sh deleted file mode 100755 index 1c9dbc7f68..0000000000 --- a/contrib/devtools/commit-script-check.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# This simple script checks for commits beginning with: scripted-diff: -# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and -# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the -# commit message. - -# The resulting script should exactly transform the previous commit into the current -# one. Any remaining diff signals an error. - -if test "x$1" = "x"; then - echo "Usage: $0 <commit>..." - exit 1 -fi - -RET=0 -PREV_BRANCH=`git name-rev --name-only HEAD` -PREV_HEAD=`git rev-parse HEAD` -for i in `git rev-list --reverse $1`; do - if git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:"; then - git checkout --quiet $i^ || exit - SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" - if test "x$SCRIPT" = "x"; then - echo "Error: missing script for: $i" - echo "Failed" - RET=1 - else - echo "Running script for: $i" - echo "$SCRIPT" - eval "$SCRIPT" - git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1 - fi - git reset --quiet --hard HEAD - else - if git rev-list "--format=%b" -n1 $i | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then - echo "Error: script block marker but no scripted-diff in title" - echo "Failed" - RET=1 - fi - fi -done -git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD -exit $RET diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index c817e794b9..da7d74bdc4 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -146,7 +146,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name): ################################################################################ def read_file(filename): - return open(os.path.abspath(filename), 'r').read() + return open(os.path.abspath(filename), 'r', encoding="utf8").read() def gather_file_info(filename): info = {} @@ -286,7 +286,7 @@ Arguments: def report_cmd(argv): if len(argv) == 2: sys.exit(REPORT_USAGE) - + base_directory = argv[2] if not os.path.exists(base_directory): sys.exit("*** bad <base_directory>: %s" % base_directory) @@ -325,13 +325,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(os.path.abspath(filename), 'r') + f = open(os.path.abspath(filename), 'r', encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(os.path.abspath(filename), 'w') + f = open(os.path.abspath(filename), 'w', encoding="utf8") f.write(''.join(file_lines)) f.close() @@ -444,7 +444,7 @@ def print_file_action_message(filename, action): def update_cmd(argv): if len(argv) != 3: sys.exit(UPDATE_USAGE) - + base_directory = argv[2] if not os.path.exists(base_directory): sys.exit("*** bad base_directory: %s" % base_directory) @@ -506,7 +506,7 @@ def file_has_hashbang(file_lines): def insert_python_header(filename, file_lines, start_year, end_year): if file_has_hashbang(file_lines): - insert_idx = 1 + insert_idx = 1 else: insert_idx = 0 header_lines = get_python_header_lines_to_insert(start_year, end_year) @@ -570,13 +570,13 @@ def insert_cmd(argv): _, extension = os.path.splitext(filename) if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: sys.exit("*** cannot insert for file extension %s" % extension) - - if extension == '.py': + + if extension == '.py': style = 'python' else: style = 'cpp' exec_insert_header(filename, style) - + ################################################################################ # UI ################################################################################ diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 27c80548c1..b5de5a395f 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,5 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash +export LC_ALL=C TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} BUILDDIR=${BUILDDIR:-$TOPDIR} diff --git a/contrib/devtools/git-subtree-check.sh b/contrib/devtools/git-subtree-check.sh deleted file mode 100755 index 184951715e..0000000000 --- a/contrib/devtools/git-subtree-check.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -DIR="$1" -COMMIT="$2" -if [ -z "$COMMIT" ]; then - COMMIT=HEAD -fi - -# Taken from git-subtree (Copyright (C) 2009 Avery Pennarun <apenwarr@gmail.com>) -find_latest_squash() -{ - dir="$1" - sq= - main= - sub= - git log --grep="^git-subtree-dir: $dir/*\$" \ - --pretty=format:'START %H%n%s%n%n%b%nEND%n' "$COMMIT" | - while read a b _; do - case "$a" in - START) sq="$b" ;; - git-subtree-mainline:) main="$b" ;; - git-subtree-split:) sub="$b" ;; - END) - if [ -n "$sub" ]; then - if [ -n "$main" ]; then - # a rejoin commit? - # Pretend its sub was a squash. - sq="$sub" - fi - echo "$sq" "$sub" - break - fi - sq= - main= - sub= - ;; - esac - done -} - -# find latest subtree update -latest_squash="$(find_latest_squash "$DIR")" -if [ -z "$latest_squash" ]; then - echo "ERROR: $DIR is not a subtree" >&2 - exit 2 -fi -set $latest_squash -old=$1 -rev=$2 - -# get the tree in the current commit -tree_actual=$(git ls-tree -d "$COMMIT" "$DIR" | head -n 1) -if [ -z "$tree_actual" ]; then - echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2 - exit 1 -fi -set $tree_actual -tree_actual_type=$2 -tree_actual_tree=$3 -echo "$DIR in $COMMIT currently refers to $tree_actual_type $tree_actual_tree" -if [ "d$tree_actual_type" != "dtree" ]; then - echo "FAIL: subtree directory $DIR is not a tree in $COMMIT" >&2 - exit 1 -fi - -# get the tree at the time of the last subtree update -tree_commit=$(git show -s --format="%T" $old) -echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)" - -# ... and compare the actual tree with it -if [ "$tree_actual_tree" != "$tree_commit" ]; then - git diff $tree_commit $tree_actual_tree >&2 - echo "FAIL: subtree directory was touched without subtree merge" >&2 - exit 1 -fi - -# get the tree in the subtree commit referred to -if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "subtree commit $rev unavailable: cannot compare" >&2 - exit -fi -tree_subtree=$(git show -s --format="%T" $rev) -echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" - -# ... and compare the actual tree with it -if [ "$tree_actual_tree" != "$tree_subtree" ]; then - echo "FAIL: subtree update commit differs from upstream tree!" >&2 - exit 1 -fi - -echo "GOOD" diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 2941d2cb6d..4e90f85f50 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -21,7 +21,8 @@ import argparse import hashlib import subprocess import sys -import json,codecs +import json +import codecs try: from urllib.request import Request,urlopen except: @@ -46,7 +47,7 @@ def git_config_get(option, default=None): ''' try: return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8') - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: return default def retrieve_pr_info(repo,pull): @@ -190,26 +191,26 @@ def main(): merge_branch = 'pull/'+pull+'/merge' local_merge_branch = 'pull/'+pull+'/local-merge' - devnull = open(os.devnull,'w') + devnull = open(os.devnull, 'w', encoding="utf8") try: subprocess.check_call([GIT,'checkout','-q',branch]) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot check out branch %s." % (branch), file=stderr) sys.exit(3) try: subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*', '+refs/heads/'+branch+':refs/heads/'+base_branch]) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot find pull request #%s or branch %s on %s." % (pull,branch,host_repo), file=stderr) sys.exit(3) try: subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr) sys.exit(3) try: subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr) sys.exit(3) subprocess.check_call([GIT,'checkout','-q',base_branch]) @@ -230,7 +231,7 @@ def main(): message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot be merged cleanly.",file=stderr) subprocess.check_call([GIT,'merge','--abort']) sys.exit(4) @@ -249,12 +250,12 @@ def main(): try: first_sha512 = tree_sha512sum() message += '\n\nTree-SHA512: ' + first_sha512 - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Unable to compute tree hash") sys.exit(4) try: subprocess.check_call([GIT,'commit','--amend','-m',message.encode('utf-8')]) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("ERROR: Cannot update message.", file=stderr) sys.exit(4) @@ -299,7 +300,7 @@ def main(): try: subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) break - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: print("Error while signing, asking again.",file=stderr) elif reply == 'x': print("Not signing off on merge, exiting.",file=stderr) diff --git a/contrib/devtools/lint-all.sh b/contrib/devtools/lint-all.sh deleted file mode 100755 index b6d86959c6..0000000000 --- a/contrib/devtools/lint-all.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit -# with a non-zero status code. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -for f in "${SCRIPTDIR}"/lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - exit 1 - fi - fi -done diff --git a/contrib/devtools/lint-python.sh b/contrib/devtools/lint-python.sh deleted file mode 100755 index e2c9d775a6..0000000000 --- a/contrib/devtools/lint-python.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -# E112 expected an indented block -# E113 unexpected indentation -# E115 expected an indented block (comment) -# E116 unexpected indentation (comment) -# E125 continuation line with same indent as next logical line -# E131 continuation line unaligned for hanging indent -# E133 closing bracket is missing indentation -# E223 tab before operator -# E224 tab after operator -# E271 multiple spaces after keyword -# E272 multiple spaces before keyword -# E273 tab after keyword -# E274 tab before keyword -# E275 missing whitespace after keyword -# E304 blank lines found after function decorator -# E306 expected 1 blank line before a nested definition -# E502 the backslash is redundant between brackets -# E702 multiple statements on one line (semicolon) -# E703 statement ends with a semicolon -# E714 test for object identity should be "is not" -# E721 do not compare types, use "isinstance()" -# E741 do not use variables named "l", "O", or "I" -# E742 do not define classes named "l", "O", or "I" -# E743 do not define functions named "l", "O", or "I" -# F401 module imported but unused -# F402 import module from line N shadowed by loop variable -# F404 future import(s) name after other statements -# F406 "from module import *" only allowed at module level -# F407 an undefined __future__ feature name was imported -# F601 dictionary key name repeated with different values -# F602 dictionary key variable name repeated with different values -# F621 too many expressions in an assignment with star-unpacking -# F622 two or more starred expressions in an assignment (a, *b, *c = d) -# F631 assertion test is a tuple, which are always True -# F701 a break statement outside of a while or for loop -# F702 a continue statement outside of a while or for loop -# F703 a continue statement in a finally block in a loop -# F704 a yield or yield from statement outside of a function -# F705 a return statement with arguments inside a generator -# F706 a return statement outside of a function/method -# F707 an except: block as not the last exception handler -# F811 redefinition of unused name from line N -# F812 list comprehension redefines 'foo' from line N -# F822 undefined name name in __all__ -# F823 local variable name … referenced before assignment -# F831 duplicate argument name in function definition -# W292 no newline at end of file -# W504 line break after binary operator -# W601 .has_key() is deprecated, use "in" -# W602 deprecated form of raising exception -# W603 "<>" is deprecated, use "!=" -# W604 backticks are deprecated, use "repr()" -# W605 invalid escape sequence "x" - -flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E271,E272,E273,E274,E275,E304,E306,E502,E702,E703,E714,E721,E741,E742,E743,F401,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F822,F823,F831,W292,W504,W601,W602,W603,W604,W605 . diff --git a/contrib/devtools/lint-whitespace.sh b/contrib/devtools/lint-whitespace.sh deleted file mode 100755 index c5d43043d5..0000000000 --- a/contrib/devtools/lint-whitespace.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace. - -# We can't run this check unless we know the commit range for the PR. - -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: .lint-whitespace.sh [N]" - echo " TRAVIS_COMMIT_RANGE='<commit range>' .lint-whitespace.sh" - echo " .lint-whitespace.sh -?" - echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' .lint-whitespace.sh" - exit 0 - ;; - esac -done - -if [ -z "${TRAVIS_COMMIT_RANGE}" ]; then - if [ "$1" ]; then - TRAVIS_COMMIT_RANGE="HEAD~$1...HEAD" - else - TRAVIS_COMMIT_RANGE="HEAD" - fi -fi - -showdiff() { - if ! git diff -U0 "${TRAVIS_COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)src/leveldb/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -showcodediff() { - if ! git diff -U0 "${TRAVIS_COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -RET=0 - -# Check if trailing whitespace was found in the diff. -if showdiff | grep -E -q '^\+.*\s+$'; then - echo "This diff appears to have added new lines with trailing whitespace." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with trailing whitespace, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)') - RET=1 -fi - -# Check if tab characters were found in the diff. -if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then - echo "This diff appears to have added new lines with tab characters instead of spaces." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with a tab character, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}') - RET=1 -fi - -exit $RET diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py index 5cb3bb6f75..a75731ef76 100755 --- a/contrib/devtools/optimize-pngs.py +++ b/contrib/devtools/optimize-pngs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ import os import sys import subprocess import hashlib -from PIL import Image +from PIL import Image # pip3 install Pillow def file_hash(filename): '''Return hash of raw file contents''' @@ -27,7 +27,7 @@ def content_hash(filename): pngcrush = 'pngcrush' git = 'git' folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"] -basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel']).rstrip('\n') +basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True).rstrip('\n') totalSaveBytes = 0 noHashChange = True @@ -37,42 +37,40 @@ for folder in folders: for file in os.listdir(absFolder): extension = os.path.splitext(file)[1] if extension.lower() == '.png': - print("optimizing "+file+"..."), + print("optimizing {}...".format(file), end =' ') file_path = os.path.join(absFolder, file) fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)} fileMetaMap['contentHashPre'] = content_hash(file_path) - - pngCrushOutput = "" + try: - pngCrushOutput = subprocess.check_output( - [pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path], - stderr=subprocess.STDOUT).rstrip('\n') + subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except: - print "pngcrush is not installed, aborting..." + print("pngcrush is not installed, aborting...") sys.exit(0) - + #verify - if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT): - print "PNG file "+file+" is corrupted after crushing, check out pngcursh version" + if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True): + print("PNG file "+file+" is corrupted after crushing, check out pngcursh version") sys.exit(1) - + fileMetaMap['sha256New'] = file_hash(file_path) fileMetaMap['contentHashPost'] = content_hash(file_path) if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']: - print "Image contents of PNG file "+file+" before and after crushing don't match" + print("Image contents of PNG file {} before and after crushing don't match".format(file)) sys.exit(1) fileMetaMap['psize'] = os.path.getsize(file_path) outputArray.append(fileMetaMap) - print("done\n"), + print("done") -print "summary:\n+++++++++++++++++" +print("summary:\n+++++++++++++++++") for fileDict in outputArray: oldHash = fileDict['sha256Old'] newHash = fileDict['sha256New'] totalSaveBytes += fileDict['osize'] - fileDict['psize'] noHashChange = noHashChange and (oldHash == newHash) - print fileDict['file']+"\n size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n old sha256: "+oldHash+"\n new sha256: "+newHash+"\n" - -print "completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes" + print(fileDict['file']+"\n size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n old sha256: "+oldHash+"\n new sha256: "+newHash+"\n") + +print("completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes") diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 1613f704df..47195f73c8 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,6 @@ Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. Needs `readelf` (for ELF) and `objdump` (for PE). ''' -from __future__ import division,print_function,unicode_literals import subprocess import sys import os @@ -21,38 +20,38 @@ def check_ELF_PIE(executable): ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): line = line.split() - if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN': + if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': ok = True return ok def get_ELF_program_headers(executable): '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') in_headers = False count = 0 headers = [] - for line in stdout.split(b'\n'): - if line.startswith(b'Program Headers:'): + for line in stdout.splitlines(): + if line.startswith('Program Headers:'): in_headers = True - if line == b'': + if line == '': in_headers = False if in_headers: if count == 1: # header line - ofs_typ = line.find(b'Type') - ofs_offset = line.find(b'Offset') - ofs_flags = line.find(b'Flg') - ofs_align = line.find(b'Align') + ofs_typ = line.find('Type') + ofs_offset = line.find('Offset') + ofs_flags = line.find('Flg') + ofs_align = line.find('Align') if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: raise ValueError('Cannot parse elfread -lW output') elif count > 1: @@ -69,9 +68,9 @@ def check_ELF_NX(executable): have_wx = False have_gnu_stack = False for (typ, flags) in get_ELF_program_headers(executable): - if typ == b'GNU_STACK': + if typ == 'GNU_STACK': have_gnu_stack = True - if b'W' in flags and b'E' in flags: # section is both writable and executable + if 'W' in flags and 'E' in flags: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx @@ -88,17 +87,17 @@ def check_ELF_RELRO(executable): # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 - if typ == b'GNU_RELRO': + if typ == 'GNU_RELRO': have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]): + if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]): have_bindnow = True return have_gnu_relro and have_bindnow @@ -106,13 +105,13 @@ def check_ELF_Canary(executable): ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False - for line in stdout.split(b'\n'): - if b'__stack_chk_fail' in line: + for line in stdout.splitlines(): + if '__stack_chk_fail' in line: ok = True return ok @@ -122,13 +121,13 @@ def get_PE_dll_characteristics(executable): Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' and bits is the DllCharacteristics value. ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') arch = '' bits = 0 - for line in stdout.split('\n'): + for line in stdout.splitlines(): tokens = line.split() if len(tokens)>=2 and tokens[0] == 'architecture:': arch = tokens[1].rstrip(',') @@ -151,7 +150,7 @@ def check_PE_DYNAMIC_BASE(executable): def check_PE_HIGH_ENTROPY_VA(executable): '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': + if arch == 'i386:x86-64': reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA else: # Unnecessary on 32-bit assert(arch == 'i386') diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 98daa1bd76..6808e77da7 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,6 @@ Example usage: find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py ''' -from __future__ import division, print_function, unicode_literals import subprocess import re import sys @@ -47,28 +46,28 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -b'_edata', b'_end', b'_init', b'__bss_start', b'_fini', b'_IO_stdin_used' +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -b'libgcc_s.so.1', # GCC base support -b'libc.so.6', # C library -b'libpthread.so.0', # threading -b'libanl.so.1', # DNS resolve -b'libm.so.6', # math library -b'librt.so.1', # real-time (clock) -b'ld-linux-x86-64.so.2', # 64-bit dynamic linker -b'ld-linux.so.2', # 32-bit dynamic linker +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker # bitcoin-qt only -b'libX11-xcb.so.1', # part of X11 -b'libX11.so.6', # part of X11 -b'libxcb.so.1', # part of X11 -b'libfontconfig.so.1', # font support -b'libfreetype.so.6', # font parsing -b'libdl.so.2' # programming interface to dynamic linker +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker } class CPPFilt(object): @@ -78,10 +77,10 @@ class CPPFilt(object): Use a pipe to the 'c++filt' command. ''' def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): - self.proc.stdin.write(mangled + b'\n') + self.proc.stdin.write(mangled + '\n') self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() @@ -95,43 +94,43 @@ def read_symbols(executable, imports=True): Parse an ELF executable and return a list of (symbol,version) tuples for dynamic, imported symbols. ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) syms = [] - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): line = line.split() - if len(line)>7 and re.match(b'[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition(b'@') - is_import = line[6] == b'UND' - if version.startswith(b'@'): + if len(line)>7 and re.match('[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition('@') + is_import = line[6] == 'UND' + if version.startswith('@'): version = version[1:] if is_import == imports: syms.append((sym, version)) return syms def check_version(max_versions, version): - if b'_' in version: - (lib, _, ver) = version.rpartition(b'_') + if '_' in version: + (lib, _, ver) = version.rpartition('_') else: lib = version ver = '0' - ver = tuple([int(x) for x in ver.split(b'.')]) + ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False return ver <= max_versions[lib] def read_libraries(filename): - p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') libraries = [] - for line in stdout.split(b'\n'): + for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>2 and tokens[1] == b'(NEEDED)': - match = re.match(b'^Shared library: \[(.*)\]$', b' '.join(tokens[2:])) + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: @@ -145,18 +144,18 @@ if __name__ == '__main__': # Check imported symbols for sym,version in read_symbols(filename, True): if version and not check_version(MAX_VERSIONS, version): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym).decode('utf-8'), version.decode('utf-8'))) + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) retval = 1 # Check exported symbols for sym,version in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym).decode('utf-8'))) + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name.decode('utf-8'))) + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 22f5ee20f7..9b6d6bf665 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,16 +1,15 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' -from __future__ import division,print_function import subprocess import unittest def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include <stdio.h> int main() @@ -22,7 +21,7 @@ def write_testcode(filename): def call_security_check(cc, source, executable, options): subprocess.check_call([cc,source,'-o',executable] + options) - p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() return (p.returncode, stdout.rstrip()) @@ -33,29 +32,39 @@ class TestSecurityChecks(unittest.TestCase): cc = 'gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), (0, '')) - def test_PE(self): + def test_32bit_PE(self): source = 'test1.c' executable = 'test1.exe' cc = 'i686-w64-mingw32-gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, []), - (1, executable+': failed PIE NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), + self.assertEqual(call_security_check(cc, source, executable, []), + (1, executable+': failed DYNAMIC_BASE NX')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), + (1, executable+': failed DYNAMIC_BASE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), (0, '')) + def test_64bit_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = 'x86_64-w64-mingw32-gcc' + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, []), (1, executable+': failed DYNAMIC_BASE NX\n'+executable+': warning HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), (1, executable+': failed DYNAMIC_BASE\n'+executable+': warning HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), (0, executable+': warning HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va']), (0, '')) if __name__ == '__main__': unittest.main() diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index e1924749d2..f0098cfcdf 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ It will do the following automatically: TODO: - auto-add new translations to the build system according to the translation process ''' -from __future__ import division, print_function import subprocess import re import sys @@ -31,6 +30,8 @@ SOURCE_LANG = 'bitcoin_en.ts' LOCALE_DIR = 'src/qt/locale' # Minimum number of messages for translation to be considered at all MIN_NUM_MESSAGES = 10 +# Regexp to check for Bitcoin addresses +ADDRESS_REGEXP = re.compile('([13]|bc1)[a-zA-Z0-9]{30,}') def check_at_repository_root(): if not os.path.exists('.git'): @@ -123,6 +124,12 @@ def escape_cdata(text): text = text.replace('"', '"') return text +def contains_bitcoin_addr(text, errors): + if text != None and ADDRESS_REGEXP.search(text) != None: + errors.append('Translation "%s" contains a bitcoin address. This will be removed.' % (text)) + return True + return False + def postprocess_translations(reduce_diff_hacks=False): print('Checking and postprocessing...') @@ -161,7 +168,7 @@ def postprocess_translations(reduce_diff_hacks=False): if translation is None: continue errors = [] - valid = check_format_specifiers(source, translation, errors, numerus) + valid = check_format_specifiers(source, translation, errors, numerus) and not contains_bitcoin_addr(translation, errors) for error in errors: print('%s: %s' % (filename, error)) diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 299377d691..df1db76e92 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -13,8 +13,8 @@ pattern = args.pattern outfile = args.outfile in_remove = False -with open(tracefile, 'r') as f: - with open(outfile, 'w') as wf: +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: for line in f: for p in pattern: if line.startswith("SF:") and p in line: diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py new file mode 100755 index 0000000000..1da9e43896 --- /dev/null +++ b/contrib/gitian-build.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + programs += ['docker.io'] + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian.sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) + if not os.path.isdir('bitcoin-detached-sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) + if not os.path.isdir('gitian-builder'): + subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('bitcoin'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) + os.chdir('gitian-builder') + make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + +def build(): + global args, workdir + + os.makedirs('bitcoin-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) + subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) + subprocess.check_call('mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../bitcoin-binaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) + subprocess.check_call('mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../bitcoin-binaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) + subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + +def sign(): + global args, workdir + os.chdir('gitian-builder') + + if args.windows: + print('\nSigning ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) + subprocess.check_call('mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/bitcoin-*win32-setup.exe ../bitcoin-binaries/'+args.version, shell=True) + + if args.macos: + print('\nSigning ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) + subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/'+args.version+'/bitcoin-'+args.version+'-osx.dmg', shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Signed Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) + subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) + os.chdir(workdir) + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) + print('\nVerifying v'+args.version+' Signed Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) + print('\nVerifying v'+args.version+' Signed MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) + + os.chdir(workdir) + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/bitcoin/bitcoin', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') + parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') + parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') + parser.add_argument('signer', help='GPG signer to sign each build assert file') + parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + if args.buildsign: + args.build=True + args.sign=True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + args.commit = ('' if args.commit else 'v') + args.version + print(args.commit) + + if args.setup: + setup() + + os.chdir('bitcoin') + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.sign: + sign() + + if args.verify: + verify() + +if __name__ == '__main__': + main() diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh deleted file mode 100755 index 94d6a89c7b..0000000000 --- a/contrib/gitian-build.sh +++ /dev/null @@ -1,389 +0,0 @@ -# Copyright (c) 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# What to do -sign=false -verify=false -build=false - -# Systems to build -linux=true -windows=true -osx=true - -# Other Basic variables -SIGNER= -VERSION= -commit=false -url=https://github.com/bitcoin/bitcoin -proc=2 -mem=2000 -lxc=true -osslTarUrl=http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz -osslPatchUrl=https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch -scriptName=$(basename -- "$0") -signProg="gpg --detach-sign" -commitFiles=true - -# Help Message -read -d '' usage <<- EOF -Usage: $scriptName [-c|u|v|b|s|B|o|h|j|m|] signer version - -Run this script from the directory containing the bitcoin, gitian-builder, gitian.sigs, and bitcoin-detached-sigs. - -Arguments: -signer GPG signer to sign each build assert file -version Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified - -Options: --c|--commit Indicate that the version argument is for a commit or branch --u|--url Specify the URL of the repository. Default is https://github.com/bitcoin/bitcoin --v|--verify Verify the Gitian build --b|--build Do a Gitian build --s|--sign Make signed binaries for Windows and Mac OSX --B|--buildsign Build both signed and unsigned binaries --o|--os Specify which Operating Systems the build is for. Default is lwx. l for linux, w for windows, x for osx --j Number of processes to use. Default 2 --m Memory to allocate in MiB. Default 2000 ---kvm Use KVM instead of LXC ---setup Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian) ---detach-sign Create the assert file for detached signing. Will not commit anything. ---no-commit Do not commit anything to git --h|--help Print this help message -EOF - -# Get options and arguments -while :; do - case $1 in - # Verify - -v|--verify) - verify=true - ;; - # Build - -b|--build) - build=true - ;; - # Sign binaries - -s|--sign) - sign=true - ;; - # Build then Sign - -B|--buildsign) - sign=true - build=true - ;; - # PGP Signer - -S|--signer) - if [ -n "$2" ] - then - SIGNER="$2" - shift - else - echo 'Error: "--signer" requires a non-empty argument.' - exit 1 - fi - ;; - # Operating Systems - -o|--os) - if [ -n "$2" ] - then - linux=false - windows=false - osx=false - if [[ "$2" = *"l"* ]] - then - linux=true - fi - if [[ "$2" = *"w"* ]] - then - windows=true - fi - if [[ "$2" = *"x"* ]] - then - osx=true - fi - shift - else - echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)' - exit 1 - fi - ;; - # Help message - -h|--help) - echo "$usage" - exit 0 - ;; - # Commit or branch - -c|--commit) - commit=true - ;; - # Number of Processes - -j) - if [ -n "$2" ] - then - proc=$2 - shift - else - echo 'Error: "-j" requires an argument' - exit 1 - fi - ;; - # Memory to allocate - -m) - if [ -n "$2" ] - then - mem=$2 - shift - else - echo 'Error: "-m" requires an argument' - exit 1 - fi - ;; - # URL - -u) - if [ -n "$2" ] - then - url=$2 - shift - else - echo 'Error: "-u" requires an argument' - exit 1 - fi - ;; - # kvm - --kvm) - lxc=false - ;; - # Detach sign - --detach-sign) - signProg="true" - commitFiles=false - ;; - # Commit files - --no-commit) - commitFiles=false - ;; - # Setup - --setup) - setup=true - ;; - *) # Default case: If no more options then break out of the loop. - break - esac - shift -done - -# Set up LXC -if [[ $lxc = true ]] -then - export USE_LXC=1 -fi - -# Check for OSX SDK -if [[ ! -e "gitian-builder/inputs/MacOSX10.11.sdk.tar.gz" && $osx == true ]] -then - echo "Cannot build for OSX, SDK does not exist. Will build for other OSes" - osx=false -fi - -# Get signer -if [[ -n "$1" ]] -then - SIGNER="$1" - shift -fi - -# Get version -if [[ -n "$1" ]] -then - VERSION=$1 - COMMIT=$VERSION - shift -fi - -# Check that a signer is specified -if [[ "$SIGNER" == "" ]] -then - echo "$scriptName: Missing signer." - echo "Try $scriptName --help for more information" - exit 1 -fi - -# Check that a version is specified -if [[ $VERSION == "" ]] -then - echo "$scriptName: Missing version." - echo "Try $scriptName --help for more information" - exit 1 -fi - -# Add a "v" if no -c -if [[ $commit = false ]] -then - COMMIT="v${VERSION}" -fi -echo ${COMMIT} - -# Setup build environment -if [[ $setup = true ]] -then - sudo apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm qemu-utils - git clone https://github.com/bitcoin-core/gitian.sigs.git - git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git - git clone https://github.com/devrandom/gitian-builder.git - pushd ./gitian-builder - if [[ -n "$USE_LXC" ]] - then - sudo apt-get install lxc - bin/make-base-vm --suite trusty --arch amd64 --lxc - else - bin/make-base-vm --suite trusty --arch amd64 - fi - popd -fi - -# Set up build -pushd ./bitcoin -git fetch -git checkout ${COMMIT} -popd - -# Build -if [[ $build = true ]] -then - # Make output folder - mkdir -p ./bitcoin-binaries/${VERSION} - - # Build Dependencies - echo "" - echo "Building Dependencies" - echo "" - pushd ./gitian-builder - mkdir -p inputs - wget -N -P inputs $osslPatchUrl - wget -N -P inputs $osslTarUrl - make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common - - # Linux - if [[ $linux = true ]] - then - echo "" - echo "Compiling ${VERSION} Linux" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/${VERSION} - fi - # Windows - if [[ $windows = true ]] - then - echo "" - echo "Compiling ${VERSION} Windows" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz - mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../bitcoin-binaries/${VERSION} - fi - # Mac OSX - if [[ $osx = true ]] - then - echo "" - echo "Compiling ${VERSION} Mac OSX" - echo "" - ./bin/gbuild -j ${proc} -m ${mem} --commit bitcoin=${COMMIT} --url bitcoin=${url} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz - mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../bitcoin-binaries/${VERSION} - fi - popd - - if [[ $commitFiles = true ]] - then - # Commit to gitian.sigs repo - echo "" - echo "Committing ${VERSION} Unsigned Sigs" - echo "" - pushd gitian.sigs - git add ${VERSION}-linux/"${SIGNER}" - git add ${VERSION}-win-unsigned/"${SIGNER}" - git add ${VERSION}-osx-unsigned/"${SIGNER}" - git commit -a -m "Add ${VERSION} unsigned sigs for ${SIGNER}" - popd - fi -fi - -# Verify the build -if [[ $verify = true ]] -then - # Linux - pushd ./gitian-builder - echo "" - echo "Verifying v${VERSION} Linux" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - # Windows - echo "" - echo "Verifying v${VERSION} Windows" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - # Mac OSX - echo "" - echo "Verifying v${VERSION} Mac OSX" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - # Signed Windows - echo "" - echo "Verifying v${VERSION} Signed Windows" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - # Signed Mac OSX - echo "" - echo "Verifying v${VERSION} Signed Mac OSX" - echo "" - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - popd -fi - -# Sign binaries -if [[ $sign = true ]] -then - - pushd ./gitian-builder - # Sign Windows - if [[ $windows = true ]] - then - echo "" - echo "Signing ${VERSION} Windows" - echo "" - ./bin/gbuild -i --commit signature=${COMMIT} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/${VERSION} - mv build/out/bitcoin-*win32-setup.exe ../bitcoin-binaries/${VERSION} - fi - # Sign Mac OSX - if [[ $osx = true ]] - then - echo "" - echo "Signing ${VERSION} Mac OSX" - echo "" - ./bin/gbuild -i --commit signature=${COMMIT} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/${VERSION}/bitcoin-${VERSION}-osx.dmg - fi - popd - - if [[ $commitFiles = true ]] - then - # Commit Sigs - pushd gitian.sigs - echo "" - echo "Committing ${VERSION} Signed Sigs" - echo "" - git add ${VERSION}-win-signed/"${SIGNER}" - git add ${VERSION}-osx-signed/"${SIGNER}" - git commit -a -m "Add ${VERSION} signed binary sigs for ${SIGNER}" - popd - fi -fi diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 3e9ee0495a..1c8aca6f65 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -2,23 +2,23 @@ name: "bitcoin-linux-0.17" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "g++-aarch64-linux-gnu" -- "g++-4.8-aarch64-linux-gnu" -- "gcc-4.8-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-7-aarch64-linux-gnu" - "binutils-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" -- "g++-4.8-arm-linux-gnueabihf" -- "gcc-4.8-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-7-arm-linux-gnueabihf" - "binutils-arm-linux-gnueabihf" -- "g++-4.8-multilib" -- "gcc-4.8-multilib" +- "g++-7-multilib" +- "gcc-7-multilib" - "binutils-gold" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "libtool" @@ -56,7 +56,7 @@ script: | function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} @@ -68,7 +68,7 @@ script: | function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} @@ -98,7 +98,7 @@ script: | for prog in gcc g++; do rm -f ${WRAP_DIR}/${prog} cat << EOF > ${WRAP_DIR}/${prog} - #!/bin/bash + #!/usr/bin/env bash REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" for var in "\$@" do diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index f6e9414ab1..297a136fae 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,7 +1,7 @@ --- name: "bitcoin-dmg-signer" suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: @@ -19,7 +19,7 @@ script: | # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index a84dce3e3a..7d4793b97d 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -2,14 +2,14 @@ name: "bitcoin-osx-0.17" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "ca-certificates" - "curl" - "g++" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "librsvg2-bin" @@ -55,7 +55,7 @@ script: | function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} @@ -67,7 +67,7 @@ script: | function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 3c1e0214a0..2f3ec3e8ff 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,7 +1,7 @@ --- name: "bitcoin-win-signer" suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 8a87d91754..9c588afcda 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -2,13 +2,13 @@ name: "bitcoin-win-0.17" enable_cache: true suites: -- "trusty" +- "bionic" architectures: - "amd64" packages: - "curl" - "g++" -- "git-core" +- "git" - "pkg-config" - "autoconf" - "libtool" @@ -21,6 +21,7 @@ packages: - "zip" - "ca-certificates" - "python" +- "rename" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -29,7 +30,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" - FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" @@ -48,7 +49,7 @@ script: | function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} @@ -60,7 +61,7 @@ script: | function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} @@ -76,15 +77,15 @@ script: | for i in $HOSTS; do mkdir -p ${WRAP_DIR}/${i} for prog in collect2; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}/${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} REAL=$(${i}-gcc -print-prog-name=${prog}) echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} chmod +x ${WRAP_DIR}/${i}/${prog} done for prog in gcc g++; do - echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-keys/README.md b/contrib/gitian-keys/README.md index a9339c8bda..ffe4fb144b 100644 --- a/contrib/gitian-keys/README.md +++ b/contrib/gitian-keys/README.md @@ -1,9 +1,10 @@ ## PGP keys of Gitian builders and Developers -The keys.txt contains the public keys of Gitian builders and active developers. +The file `keys.txt` contains fingerprints of the public keys of Gitian builders +and active developers. -The keys are mainly used to sign git commits or the build results of Gitian -builds. +The associated keys are mainly used to sign git commits or the build results +of Gitian builds. The most recent version of each pgp key can be found on most pgp key servers. diff --git a/contrib/init/README.md b/contrib/init/README.md index 1a949f3c07..8d3e57c526 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -5,7 +5,7 @@ Upstart: bitcoind.conf OpenRC: bitcoind.openrc bitcoind.openrcconf CentOS: bitcoind.init -OS X: org.bitcoin.bitcoind.plist +macOS: org.bitcoin.bitcoind.plist ``` have been made available to assist packagers in creating node packages here. diff --git a/contrib/init/bitcoind.init b/contrib/init/bitcoind.init index db5061874b..0c95baf3a1 100644 --- a/contrib/init/bitcoind.init +++ b/contrib/init/bitcoind.init @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # bitcoind The bitcoin core server. # diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index d315a7d3b7..4f74e67f2f 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -2,6 +2,7 @@ # Install libdb4.8 (Berkeley DB). +export LC_ALL=C set -e if [ -z "${1}" ]; then diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 4969e96827..b501388fd2 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -21,302 +21,301 @@ from binascii import hexlify, unhexlify settings = {} -##### Switch endian-ness ##### def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() def uint32(x): - return x & 0xffffffff + return x & 0xffffffff def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) + return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | + (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + word = struct.unpack('@I', in_buf[i:i+4])[0] + out_words.append(struct.pack('@I', bytereverse(word))) + return b''.join(out_words) def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + out_words.append(in_buf[i:i+4]) + out_words.reverse() + return b''.join(out_words) def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() + hash1 = hashlib.sha256() + hash1.update(blk_hdr) + hash1_o = hash1.digest() - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() + hash2 = hashlib.sha256() + hash2.update(hash1_o) + hash2_o = hash2.digest() - return hash2_o + return hash2_o def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') - return hash_str + hash = calc_hdr_hash(blk_hdr) + hash = bufreverse(hash) + hash = wordreverse(hash) + hash_str = hexlify(hash).decode('utf-8') + return hash_str def get_blk_dt(blk_hdr): - members = struct.unpack("<I", blk_hdr[68:68+4]) - nTime = members[0] - dt = datetime.datetime.fromtimestamp(nTime) - dt_ym = datetime.datetime(dt.year, dt.month, 1) - return (dt_ym, nTime) + members = struct.unpack("<I", blk_hdr[68:68+4]) + nTime = members[0] + dt = datetime.datetime.fromtimestamp(nTime) + dt_ym = datetime.datetime(dt.year, dt.month, 1) + return (dt_ym, nTime) # When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): - blkindex = [] - f = open(settings['hashlist'], "r") - for line in f: - line = line.rstrip() - if settings['rev_hash_bytes'] == 'true': - line = hex_switchEndian(line) - blkindex.append(line) + blkindex = [] + f = open(settings['hashlist'], "r", encoding="utf8") + for line in f: + line = line.rstrip() + if settings['rev_hash_bytes'] == 'true': + line = hex_switchEndian(line) + blkindex.append(line) - print("Read " + str(len(blkindex)) + " hashes") + print("Read " + str(len(blkindex)) + " hashes") - return blkindex + return blkindex # The block map shouldn't give or receive byte-reversed hashes. def mkblockmap(blkindex): - blkmap = {} - for height,hash in enumerate(blkindex): - blkmap[hash] = height - return blkmap + blkmap = {} + for height,hash in enumerate(blkindex): + blkmap[hash] = height + return blkmap # Block header and extent on disk BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size']) class BlockDataCopier: - def __init__(self, settings, blkindex, blkmap): - self.settings = settings - self.blkindex = blkindex - self.blkmap = blkmap - - self.inFn = 0 - self.inF = None - self.outFn = 0 - self.outsz = 0 - self.outF = None - self.outFname = None - self.blkCountIn = 0 - self.blkCountOut = 0 - - self.lastDate = datetime.datetime(2000, 1, 1) - self.highTS = 1408893517 - 315360000 - self.timestampSplit = False - self.fileOutput = True - self.setFileTime = False - self.maxOutSz = settings['max_out_sz'] - if 'output' in settings: - self.fileOutput = False - if settings['file_timestamp'] != 0: - self.setFileTime = True - if settings['split_timestamp'] != 0: - self.timestampSplit = True - # Extents and cache for out-of-order blocks - self.blockExtents = {} - self.outOfOrderData = {} - self.outOfOrderSize = 0 # running total size for items in outOfOrderData - - def writeBlock(self, inhdr, blk_hdr, rawblock): - blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) - if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 - - (blkDate, blkTS) = get_blk_dt(blk_hdr) - if self.timestampSplit and (blkDate > self.lastDate): - print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) - self.lastDate = blkDate - if self.outF: - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 - - if not self.outF: - if self.fileOutput: - self.outFname = self.settings['output_file'] - else: - self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) - print("Output file " + self.outFname) - self.outF = open(self.outFname, "wb") - - self.outF.write(inhdr) - self.outF.write(blk_hdr) - self.outF.write(rawblock) - self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) - - self.blkCountOut = self.blkCountOut + 1 - if blkTS > self.highTS: - self.highTS = blkTS - - if (self.blkCountOut % 1000) == 0: - print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % - (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) - - def inFileName(self, fn): - return os.path.join(self.settings['input'], "blk%05d.dat" % fn) - - def fetchBlock(self, extent): - '''Fetch block contents from disk given extents''' - with open(self.inFileName(extent.fn), "rb") as f: - f.seek(extent.offset) - return f.read(extent.size) - - def copyOneBlock(self): - '''Find the next block to be written in the input, and copy it to the output.''' - extent = self.blockExtents.pop(self.blkCountOut) - if self.blkCountOut in self.outOfOrderData: - # If the data is cached, use it from memory and remove from the cache - rawblock = self.outOfOrderData.pop(self.blkCountOut) - self.outOfOrderSize -= len(rawblock) - else: # Otherwise look up data on disk - rawblock = self.fetchBlock(extent) - - self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) - - def run(self): - while self.blkCountOut < len(self.blkindex): - if not self.inF: - fname = self.inFileName(self.inFn) - print("Input file " + fname) - try: - self.inF = open(fname, "rb") - except IOError: - print("Premature end of block data") - return - - inhdr = self.inF.read(8) - if (not inhdr or (inhdr[0] == "\0")): - self.inF.close() - self.inF = None - self.inFn = self.inFn + 1 - continue - - inMagic = inhdr[:4] - if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) - return - inLenLE = inhdr[4:] - su = struct.unpack("<I", inLenLE) - inLen = su[0] - 80 # length without header - blk_hdr = self.inF.read(80) - inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) - - self.hash_str = calc_hash_str(blk_hdr) - if not self.hash_str in blkmap: - # Because blocks can be written to files out-of-order as of 0.10, the script - # may encounter blocks it doesn't know about. Treat as debug output. - if settings['debug_output'] == 'true': - print("Skipping unknown block " + self.hash_str) - self.inF.seek(inLen, os.SEEK_CUR) - continue - - blkHeight = self.blkmap[self.hash_str] - self.blkCountIn += 1 - - if self.blkCountOut == blkHeight: - # If in-order block, just copy - rawblock = self.inF.read(inLen) - self.writeBlock(inhdr, blk_hdr, rawblock) - - # See if we can catch up to prior out-of-order blocks - while self.blkCountOut in self.blockExtents: - self.copyOneBlock() - - else: # If out-of-order, skip over block data for now - self.blockExtents[blkHeight] = inExtent - if self.outOfOrderSize < self.settings['out_of_order_cache_sz']: - # If there is space in the cache, read the data - # Reading the data in file sequence instead of seeking and fetching it later is preferred, - # but we don't want to fill up memory - self.outOfOrderData[blkHeight] = self.inF.read(inLen) - self.outOfOrderSize += inLen - else: # If no space in cache, seek forward - self.inF.seek(inLen, os.SEEK_CUR) - - print("Done (%i blocks written)" % (self.blkCountOut)) + def __init__(self, settings, blkindex, blkmap): + self.settings = settings + self.blkindex = blkindex + self.blkmap = blkmap + + self.inFn = 0 + self.inF = None + self.outFn = 0 + self.outsz = 0 + self.outF = None + self.outFname = None + self.blkCountIn = 0 + self.blkCountOut = 0 + + self.lastDate = datetime.datetime(2000, 1, 1) + self.highTS = 1408893517 - 315360000 + self.timestampSplit = False + self.fileOutput = True + self.setFileTime = False + self.maxOutSz = settings['max_out_sz'] + if 'output' in settings: + self.fileOutput = False + if settings['file_timestamp'] != 0: + self.setFileTime = True + if settings['split_timestamp'] != 0: + self.timestampSplit = True + # Extents and cache for out-of-order blocks + self.blockExtents = {} + self.outOfOrderData = {} + self.outOfOrderSize = 0 # running total size for items in outOfOrderData + + def writeBlock(self, inhdr, blk_hdr, rawblock): + blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) + if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 + + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if self.timestampSplit and (blkDate > self.lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + self.lastDate = blkDate + if self.outF: + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 + + if not self.outF: + if self.fileOutput: + self.outFname = self.settings['output_file'] + else: + self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) + print("Output file " + self.outFname) + self.outF = open(self.outFname, "wb") + + self.outF.write(inhdr) + self.outF.write(blk_hdr) + self.outF.write(rawblock) + self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) + + self.blkCountOut = self.blkCountOut + 1 + if blkTS > self.highTS: + self.highTS = blkTS + + if (self.blkCountOut % 1000) == 0: + print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % + (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) + + def inFileName(self, fn): + return os.path.join(self.settings['input'], "blk%05d.dat" % fn) + + def fetchBlock(self, extent): + '''Fetch block contents from disk given extents''' + with open(self.inFileName(extent.fn), "rb") as f: + f.seek(extent.offset) + return f.read(extent.size) + + def copyOneBlock(self): + '''Find the next block to be written in the input, and copy it to the output.''' + extent = self.blockExtents.pop(self.blkCountOut) + if self.blkCountOut in self.outOfOrderData: + # If the data is cached, use it from memory and remove from the cache + rawblock = self.outOfOrderData.pop(self.blkCountOut) + self.outOfOrderSize -= len(rawblock) + else: # Otherwise look up data on disk + rawblock = self.fetchBlock(extent) + + self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) + + def run(self): + while self.blkCountOut < len(self.blkindex): + if not self.inF: + fname = self.inFileName(self.inFn) + print("Input file " + fname) + try: + self.inF = open(fname, "rb") + except IOError: + print("Premature end of block data") + return + + inhdr = self.inF.read(8) + if (not inhdr or (inhdr[0] == "\0")): + self.inF.close() + self.inF = None + self.inFn = self.inFn + 1 + continue + + inMagic = inhdr[:4] + if (inMagic != self.settings['netmagic']): + print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) + return + inLenLE = inhdr[4:] + su = struct.unpack("<I", inLenLE) + inLen = su[0] - 80 # length without header + blk_hdr = self.inF.read(80) + inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) + + self.hash_str = calc_hash_str(blk_hdr) + if not self.hash_str in blkmap: + # Because blocks can be written to files out-of-order as of 0.10, the script + # may encounter blocks it doesn't know about. Treat as debug output. + if settings['debug_output'] == 'true': + print("Skipping unknown block " + self.hash_str) + self.inF.seek(inLen, os.SEEK_CUR) + continue + + blkHeight = self.blkmap[self.hash_str] + self.blkCountIn += 1 + + if self.blkCountOut == blkHeight: + # If in-order block, just copy + rawblock = self.inF.read(inLen) + self.writeBlock(inhdr, blk_hdr, rawblock) + + # See if we can catch up to prior out-of-order blocks + while self.blkCountOut in self.blockExtents: + self.copyOneBlock() + + else: # If out-of-order, skip over block data for now + self.blockExtents[blkHeight] = inExtent + if self.outOfOrderSize < self.settings['out_of_order_cache_sz']: + # If there is space in the cache, read the data + # Reading the data in file sequence instead of seeking and fetching it later is preferred, + # but we don't want to fill up memory + self.outOfOrderData[blkHeight] = self.inF.read(inLen) + self.outOfOrderSize += inLen + else: # If no space in cache, seek forward + self.inF.seek(inLen, os.SEEK_CUR) + + print("Done (%i blocks written)" % (self.blkCountOut)) if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: linearize-data.py CONFIG-FILE") - sys.exit(1) - - f = open(sys.argv[1]) - for line in f: - # skip comment lines - m = re.search('^\s*#', line) - if m: - continue - - # parse key=value lines - m = re.search('^(\w+)\s*=\s*(\S.*)$', line) - if m is None: - continue - settings[m.group(1)] = m.group(2) - f.close() - - # Force hash byte format setting to be lowercase to make comparisons easier. - # Also place upfront in case any settings need to know about it. - if 'rev_hash_bytes' not in settings: - settings['rev_hash_bytes'] = 'false' - settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() - - if 'netmagic' not in settings: - settings['netmagic'] = 'f9beb4d9' - if 'genesis' not in settings: - settings['genesis'] = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' - if 'input' not in settings: - settings['input'] = 'input' - if 'hashlist' not in settings: - settings['hashlist'] = 'hashlist.txt' - if 'file_timestamp' not in settings: - settings['file_timestamp'] = 0 - if 'split_timestamp' not in settings: - settings['split_timestamp'] = 0 - if 'max_out_sz' not in settings: - settings['max_out_sz'] = 1000 * 1000 * 1000 - if 'out_of_order_cache_sz' not in settings: - settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 - if 'debug_output' not in settings: - settings['debug_output'] = 'false' - - settings['max_out_sz'] = int(settings['max_out_sz']) - settings['split_timestamp'] = int(settings['split_timestamp']) - settings['file_timestamp'] = int(settings['file_timestamp']) - settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) - settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) - settings['debug_output'] = settings['debug_output'].lower() - - if 'output_file' not in settings and 'output' not in settings: - print("Missing output file / directory") - sys.exit(1) - - blkindex = get_block_hashes(settings) - blkmap = mkblockmap(blkindex) - - # Block hash map won't be byte-reversed. Neither should the genesis hash. - if not settings['genesis'] in blkmap: - print("Genesis block not found in hashlist") - else: - BlockDataCopier(settings, blkindex, blkmap).run() + if len(sys.argv) != 2: + print("Usage: linearize-data.py CONFIG-FILE") + sys.exit(1) + + f = open(sys.argv[1], encoding="utf8") + for line in f: + # skip comment lines + m = re.search('^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search('^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) + f.close() + + # Force hash byte format setting to be lowercase to make comparisons easier. + # Also place upfront in case any settings need to know about it. + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + + if 'netmagic' not in settings: + settings['netmagic'] = 'f9beb4d9' + if 'genesis' not in settings: + settings['genesis'] = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + if 'input' not in settings: + settings['input'] = 'input' + if 'hashlist' not in settings: + settings['hashlist'] = 'hashlist.txt' + if 'file_timestamp' not in settings: + settings['file_timestamp'] = 0 + if 'split_timestamp' not in settings: + settings['split_timestamp'] = 0 + if 'max_out_sz' not in settings: + settings['max_out_sz'] = 1000 * 1000 * 1000 + if 'out_of_order_cache_sz' not in settings: + settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 + if 'debug_output' not in settings: + settings['debug_output'] = 'false' + + settings['max_out_sz'] = int(settings['max_out_sz']) + settings['split_timestamp'] = int(settings['split_timestamp']) + settings['file_timestamp'] = int(settings['file_timestamp']) + settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) + settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) + settings['debug_output'] = settings['debug_output'].lower() + + if 'output_file' not in settings and 'output' not in settings: + print("Missing output file / directory") + sys.exit(1) + + blkindex = get_block_hashes(settings) + blkmap = mkblockmap(blkindex) + + # Block hash map won't be byte-reversed. Neither should the genesis hash. + if not settings['genesis'] in blkmap: + print("Genesis block not found in hashlist") + else: + BlockDataCopier(settings, blkindex, blkmap).run() diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 6b69c5b3a3..bfd2171947 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -21,137 +21,136 @@ import os.path settings = {} -##### Switch endian-ness ##### def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() class BitcoinRPC: - def __init__(self, host, port, username, password): - authpair = "%s:%s" % (username, password) - authpair = authpair.encode('utf-8') - self.authhdr = b"Basic " + base64.b64encode(authpair) - self.conn = httplib.HTTPConnection(host, port=port, timeout=30) - - def execute(self, obj): - try: - self.conn.request('POST', '/', json.dumps(obj), - { 'Authorization' : self.authhdr, - 'Content-type' : 'application/json' }) - except ConnectionRefusedError: - print('RPC connection refused. Check RPC settings and the server status.', - file=sys.stderr) - return None - - resp = self.conn.getresponse() - if resp is None: - print("JSON-RPC: no response", file=sys.stderr) - return None - - body = resp.read().decode('utf-8') - resp_obj = json.loads(body) - return resp_obj - - @staticmethod - def build_request(idx, method, params): - obj = { 'version' : '1.1', - 'method' : method, - 'id' : idx } - if params is None: - obj['params'] = [] - else: - obj['params'] = params - return obj - - @staticmethod - def response_is_error(resp_obj): - return 'error' in resp_obj and resp_obj['error'] is not None + def __init__(self, host, port, username, password): + authpair = "%s:%s" % (username, password) + authpair = authpair.encode('utf-8') + self.authhdr = b"Basic " + base64.b64encode(authpair) + self.conn = httplib.HTTPConnection(host, port=port, timeout=30) + + def execute(self, obj): + try: + self.conn.request('POST', '/', json.dumps(obj), + { 'Authorization' : self.authhdr, + 'Content-type' : 'application/json' }) + except ConnectionRefusedError: + print('RPC connection refused. Check RPC settings and the server status.', + file=sys.stderr) + return None + + resp = self.conn.getresponse() + if resp is None: + print("JSON-RPC: no response", file=sys.stderr) + return None + + body = resp.read().decode('utf-8') + resp_obj = json.loads(body) + return resp_obj + + @staticmethod + def build_request(idx, method, params): + obj = { 'version' : '1.1', + 'method' : method, + 'id' : idx } + if params is None: + obj['params'] = [] + else: + obj['params'] = params + return obj + + @staticmethod + def response_is_error(resp_obj): + return 'error' in resp_obj and resp_obj['error'] is not None def get_block_hashes(settings, max_blocks_per_call=10000): - rpc = BitcoinRPC(settings['host'], settings['port'], - settings['rpcuser'], settings['rpcpassword']) - - height = settings['min_height'] - while height < settings['max_height']+1: - num_blocks = min(settings['max_height']+1-height, max_blocks_per_call) - batch = [] - for x in range(num_blocks): - batch.append(rpc.build_request(x, 'getblockhash', [height + x])) - - reply = rpc.execute(batch) - if reply is None: - print('Cannot continue. Program will halt.') - return None - - for x,resp_obj in enumerate(reply): - if rpc.response_is_error(resp_obj): - print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) - sys.exit(1) - assert(resp_obj['id'] == x) # assume replies are in-sequence - if settings['rev_hash_bytes'] == 'true': - resp_obj['result'] = hex_switchEndian(resp_obj['result']) - print(resp_obj['result']) - - height += num_blocks + rpc = BitcoinRPC(settings['host'], settings['port'], + settings['rpcuser'], settings['rpcpassword']) + + height = settings['min_height'] + while height < settings['max_height']+1: + num_blocks = min(settings['max_height']+1-height, max_blocks_per_call) + batch = [] + for x in range(num_blocks): + batch.append(rpc.build_request(x, 'getblockhash', [height + x])) + + reply = rpc.execute(batch) + if reply is None: + print('Cannot continue. Program will halt.') + return None + + for x,resp_obj in enumerate(reply): + if rpc.response_is_error(resp_obj): + print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) + sys.exit(1) + assert(resp_obj['id'] == x) # assume replies are in-sequence + if settings['rev_hash_bytes'] == 'true': + resp_obj['result'] = hex_switchEndian(resp_obj['result']) + print(resp_obj['result']) + + height += num_blocks def get_rpc_cookie(): - # Open the cookie file - with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: - combined = f.readline() - combined_split = combined.split(":") - settings['rpcuser'] = combined_split[0] - settings['rpcpassword'] = combined_split[1] + # Open the cookie file + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: + combined = f.readline() + combined_split = combined.split(":") + settings['rpcuser'] = combined_split[0] + settings['rpcpassword'] = combined_split[1] if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: linearize-hashes.py CONFIG-FILE") - sys.exit(1) - - f = open(sys.argv[1]) - for line in f: - # skip comment lines - m = re.search('^\s*#', line) - if m: - continue - - # parse key=value lines - m = re.search('^(\w+)\s*=\s*(\S.*)$', line) - if m is None: - continue - settings[m.group(1)] = m.group(2) - f.close() - - if 'host' not in settings: - settings['host'] = '127.0.0.1' - if 'port' not in settings: - settings['port'] = 8332 - if 'min_height' not in settings: - settings['min_height'] = 0 - if 'max_height' not in settings: - settings['max_height'] = 313000 - if 'rev_hash_bytes' not in settings: - settings['rev_hash_bytes'] = 'false' - - use_userpass = True - use_datadir = False - if 'rpcuser' not in settings or 'rpcpassword' not in settings: - use_userpass = False - if 'datadir' in settings and not use_userpass: - use_datadir = True - if not use_userpass and not use_datadir: - print("Missing datadir or username and/or password in cfg file", file=sys.stderr) - sys.exit(1) - - settings['port'] = int(settings['port']) - settings['min_height'] = int(settings['min_height']) - settings['max_height'] = int(settings['max_height']) - - # Force hash byte format setting to be lowercase to make comparisons easier. - settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() - - # Get the rpc user and pass from the cookie if the datadir is set - if use_datadir: - get_rpc_cookie() - - get_block_hashes(settings) + if len(sys.argv) != 2: + print("Usage: linearize-hashes.py CONFIG-FILE") + sys.exit(1) + + f = open(sys.argv[1], encoding="utf8") + for line in f: + # skip comment lines + m = re.search('^\s*#', line) + if m: + continue + + # parse key=value lines + m = re.search('^(\w+)\s*=\s*(\S.*)$', line) + if m is None: + continue + settings[m.group(1)] = m.group(2) + f.close() + + if 'host' not in settings: + settings['host'] = '127.0.0.1' + if 'port' not in settings: + settings['port'] = 8332 + if 'min_height' not in settings: + settings['min_height'] = 0 + if 'max_height' not in settings: + settings['max_height'] = 313000 + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' + + use_userpass = True + use_datadir = False + if 'rpcuser' not in settings or 'rpcpassword' not in settings: + use_userpass = False + if 'datadir' in settings and not use_userpass: + use_datadir = True + if not use_userpass and not use_datadir: + print("Missing datadir or username and/or password in cfg file", file=sys.stderr) + sys.exit(1) + + settings['port'] = int(settings['port']) + settings['min_height'] = int(settings['min_height']) + settings['max_height'] = int(settings['max_height']) + + # Force hash byte format setting to be lowercase to make comparisons easier. + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + + # Get the rpc user and pass from the cookie if the datadir is set + if use_datadir: + get_rpc_cookie() + + get_block_hashes(settings) diff --git a/contrib/macdeploy/custom_dsstore.py b/contrib/macdeploy/custom_dsstore.py index e6ecabace1..b29fc71765 100755 --- a/contrib/macdeploy/custom_dsstore.py +++ b/contrib/macdeploy/custom_dsstore.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division,print_function,unicode_literals import biplist from ds_store import DSStore from mac_alias import Alias diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index 91674a92e6..f8503e4de8 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e UNSIGNED="$1" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 3379a4599c..5281ebcc47 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e ROOTDIR=dist diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index ff9fbd58df..4c175156f4 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -1,8 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e INPUTFILE="Xcode_7.3.1.dmg" diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 23a568ad13..17ce6c44f9 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -1,5 +1,4 @@ -#!/usr/bin/env python -from __future__ import division, print_function, unicode_literals +#!/usr/bin/env python3 # # Copyright (C) 2011 Patrick "p2k" Schneider <me@p2k-network.org> # @@ -203,7 +202,7 @@ def getFrameworks(binaryPath, verbose): if verbose >= 3: print("Inspecting with otool: " + binaryPath) otoolbin=os.getenv("OTOOL", "otool") - otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) o_stdout, o_stderr = otool.communicate() if otool.returncode != 0: if verbose >= 1: @@ -211,7 +210,7 @@ def getFrameworks(binaryPath, verbose): sys.stderr.flush() raise RuntimeError("otool failed with return code %d" % otool.returncode) - otoolLines = o_stdout.decode().split("\n") + otoolLines = o_stdout.split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. @@ -714,22 +713,6 @@ elif config.sign: if config.dmg is not None: - #Patch in check_output for Python 2.6 - if "check_output" not in dir( subprocess ): - def f(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return output - subprocess.check_output = f - def runHDIUtil(verb, image_basename, **kwargs): hdiutil_args = ["hdiutil", verb, image_basename + ".dmg"] if "capture_stdout" in kwargs: @@ -747,7 +730,7 @@ if config.dmg is not None: if not value is True: hdiutil_args.append(str(value)) - return run(hdiutil_args) + return run(hdiutil_args, universal_newlines=True) if verbose >= 2: if fancy is None: @@ -789,7 +772,7 @@ if config.dmg is not None: except subprocess.CalledProcessError as e: sys.exit(e.returncode) - m = re.search("/Volumes/(.+$)", output.decode()) + m = re.search("/Volumes/(.+$)", output) disk_root = m.group(0) disk_name = m.group(1) diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 0d1dd65b4f..738ea70dbe 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -2,6 +2,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C #network interface on which to limit traffic IF="eth0" #limit of the network interface in question diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index 2790ef4acd..fe7cd1d597 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -11,7 +11,7 @@ argument: nodes_main.txt nodes_test.txt -These files must consist of lines in the format +These files must consist of lines in the format <ip> <ip>:<port> @@ -34,7 +34,8 @@ These should be pasted into `src/chainparamsseeds.h`. from base64 import b32decode from binascii import a2b_hex -import sys, os +import sys +import os import re # ipv4 in ipv6 prefix @@ -46,7 +47,7 @@ def name_to_ipv6(addr): if len(addr)>6 and addr.endswith('.onion'): vchAddr = b32decode(addr[0:-6], True) if len(vchAddr) != 16-len(pchOnionCat): - raise ValueError('Invalid onion %s' % s) + raise ValueError('Invalid onion %s' % vchAddr) return pchOnionCat + vchAddr elif '.' in addr: # IPv4 return pchIPv4 + bytearray((int(x) for x in addr.split('.'))) @@ -126,13 +127,13 @@ def main(): g.write(' * Each line contains a 16-byte IPv6 address and a port.\n') g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n') g.write(' */\n') - with open(os.path.join(indir,'nodes_main.txt'),'r') as f: + with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_main', 8333) g.write('\n') - with open(os.path.join(indir,'nodes_test.txt'),'r') as f: + with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_test', 18333) g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') - + if __name__ == '__main__': main() diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 6e253c994d..59044e701a 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -6,6 +6,11 @@ # Generate seeds.txt from Pieter's DNS seeder # +import re +import sys +import dns.resolver +import collections + NSEEDS=512 MAX_SEEDS_PER_ASN=2 @@ -22,11 +27,6 @@ SUSPICIOUS_HOSTS = { "54.94.195.96", "54.94.200.247" } -import re -import sys -import dns.resolver -import collections - PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$") diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py index 816d40b49c..071bc722b0 100644 --- a/contrib/testgen/base58.py +++ b/contrib/testgen/base58.py @@ -28,7 +28,9 @@ def b58encode(v): """ long_value = 0 for (i, c) in enumerate(v[::-1]): - long_value += (256**i) * ord(c) + if isinstance(c, str): + c = ord(c) + long_value += (256**i) * c result = '' while long_value >= __b58base: @@ -41,8 +43,10 @@ def b58encode(v): # leading 0-bytes in the input become leading-1s nPad = 0 for c in v: - if c == '\0': nPad += 1 - else: break + if c == 0: + nPad += 1 + else: + break return (__b58chars[0]*nPad) + result @@ -50,8 +54,10 @@ def b58decode(v, length = None): """ decode v into a string of len bytes """ long_value = 0 - for (i, c) in enumerate(v[::-1]): - long_value += __b58chars.find(c) * (__b58base**i) + for i, c in enumerate(v[::-1]): + pos = __b58chars.find(c) + assert pos != -1 + long_value += pos * (__b58base**i) result = bytes() while long_value >= 256: @@ -62,10 +68,12 @@ def b58decode(v, length = None): nPad = 0 for c in v: - if c == __b58chars[0]: nPad += 1 - else: break + if c == __b58chars[0]: + nPad += 1 + continue + break - result = chr(0)*nPad + result + result = bytes(nPad) + result if length is not None and len(result) != length: return None @@ -92,7 +100,8 @@ def b58decode_chk(v): def get_bcaddress_version(strAddress): """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ addr = b58decode_chk(strAddress) - if addr is None or len(addr)!=21: return None + if addr is None or len(addr)!=21: + return None version = addr[0] return ord(version) diff --git a/contrib/testgen/gen_base58_test_vectors.py b/contrib/testgen/gen_base58_test_vectors.py index 8e6a5d5819..de15657d27 100755 --- a/contrib/testgen/gen_base58_test_vectors.py +++ b/contrib/testgen/gen_base58_test_vectors.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2012-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Generate valid and invalid base58 address and private key test vectors. -Usage: +Usage: gen_base58_test_vectors.py valid 50 > ../../src/test/data/base58_keys_valid.json gen_base58_test_vectors.py invalid 50 > ../../src/test/data/base58_keys_invalid.json ''' @@ -46,8 +46,8 @@ def is_valid(v): if result is None: return False for template in templates: - prefix = str(bytearray(template[0])) - suffix = str(bytearray(template[2])) + prefix = bytearray(template[0]) + suffix = bytearray(template[2]) if result.startswith(prefix) and result.endswith(suffix): if (len(result) - len(prefix) - len(suffix)) == template[1]: return True @@ -57,30 +57,33 @@ def gen_valid_vectors(): '''Generate valid test vectors''' while True: for template in templates: - prefix = str(bytearray(template[0])) - payload = os.urandom(template[1]) - suffix = str(bytearray(template[2])) + prefix = bytearray(template[0]) + payload = bytearray(os.urandom(template[1])) + suffix = bytearray(template[2]) rv = b58encode_chk(prefix + payload + suffix) assert is_valid(rv) - metadata = dict([(x,y) for (x,y) in zip(metadata_keys,template[3]) if y is not None]) - yield (rv, b2a_hex(payload), metadata) + metadata = {x: y for x, y in zip(metadata_keys,template[3]) if y is not None} + hexrepr = b2a_hex(payload) + if isinstance(hexrepr, bytes): + hexrepr = hexrepr.decode('utf8') + yield (rv, hexrepr, metadata) def gen_invalid_vector(template, corrupt_prefix, randomize_payload_size, corrupt_suffix): '''Generate possibly invalid vector''' if corrupt_prefix: prefix = os.urandom(1) else: - prefix = str(bytearray(template[0])) - + prefix = bytearray(template[0]) + if randomize_payload_size: payload = os.urandom(max(int(random.expovariate(0.5)), 50)) else: payload = os.urandom(template[1]) - + if corrupt_suffix: suffix = os.urandom(len(template[2])) else: - suffix = str(bytearray(template[2])) + suffix = bytearray(template[2]) return b58encode_chk(prefix + payload + suffix) @@ -111,7 +114,8 @@ def gen_invalid_vectors(): yield val, if __name__ == '__main__': - import sys, json + import sys + import json iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} try: uiter = iters[sys.argv[1]] @@ -121,7 +125,7 @@ if __name__ == '__main__': count = int(sys.argv[2]) except IndexError: count = 0 - + data = list(islice(uiter(), count)) json.dump(data, sys.stdout, sort_keys=True, indent=4) sys.stdout.write('\n') diff --git a/contrib/tidy_datadir.sh b/contrib/tidy_datadir.sh deleted file mode 100755 index b845b34e41..0000000000 --- a/contrib/tidy_datadir.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (c) 2013 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -if [ -d "$1" ]; then - cd "$1" || exit 1 -else - echo "Usage: $0 <datadir>" >&2 - echo "Removes obsolete Bitcoin database files" >&2 - exit 1 -fi - -LEVEL=0 -if [ -f wallet.dat -a -f addr.dat -a -f blkindex.dat -a -f blk0001.dat ]; then LEVEL=1; fi -if [ -f wallet.dat -a -f peers.dat -a -f blkindex.dat -a -f blk0001.dat ]; then LEVEL=2; fi -if [ -f wallet.dat -a -f peers.dat -a -f coins/CURRENT -a -f blktree/CURRENT -a -f blocks/blk00000.dat ]; then LEVEL=3; fi -if [ -f wallet.dat -a -f peers.dat -a -f chainstate/CURRENT -a -f blocks/index/CURRENT -a -f blocks/blk00000.dat ]; then LEVEL=4; fi - -case $LEVEL in - 0) - echo "Error: no Bitcoin datadir detected." - exit 1 - ;; - 1) - echo "Detected old Bitcoin datadir (before 0.7)." - echo "Nothing to do." - exit 0 - ;; - 2) - echo "Detected Bitcoin 0.7 datadir." - ;; - 3) - echo "Detected Bitcoin pre-0.8 datadir." - ;; - 4) - echo "Detected Bitcoin 0.8 datadir." - ;; -esac - -FILES="" -DIRS="" - -if [ $LEVEL -ge 3 ]; then FILES=$(echo $FILES blk????.dat blkindex.dat); fi -if [ $LEVEL -ge 2 ]; then FILES=$(echo $FILES addr.dat); fi -if [ $LEVEL -ge 4 ]; then DIRS=$(echo $DIRS coins blktree); fi - -for FILE in $FILES; do - if [ -f $FILE ]; then - echo "Deleting: $FILE" - rm -f $FILE - fi -done - -for DIR in $DIRS; do - if [ -d $DIR ]; then - echo "Deleting: $DIR/" - rm -rf $DIR - fi -done - -echo "Done." diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index e9e3f65da2..aa805ad1b9 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -7,20 +7,41 @@ are PGP signed (nearly always merge commits), as well as a script to verify commits against a trusted keys list. -Using verify-commits.sh safely +Using verify-commits.py safely ------------------------------ Remember that you can't use an untrusted script to verify itself. This means -that checking out code, then running `verify-commits.sh` against `HEAD` is -_not_ safe, because the version of `verify-commits.sh` that you just ran could +that checking out code, then running `verify-commits.py` against `HEAD` is +_not_ safe, because the version of `verify-commits.py` that you just ran could be backdoored. Instead, you need to use a trusted version of verify-commits prior to checkout to make sure you're checking out only code signed by trusted keys: git fetch origin && \ - ./contrib/verify-commits/verify-commits.sh origin/master && \ + ./contrib/verify-commits/verify-commits.py origin/master && \ git checkout origin/master Note that the above isn't a good UI/UX yet, and needs significant improvements to make it more convenient and reduce the chance of errors; pull-reqs improving this process would be much appreciated. + +Configuration files +------------------- + +* `trusted-git-root`: This file should contain a single git commit hash which is the first unsigned git commit (hence it is the "root of trust"). +* `trusted-sha512-root-commit`: This file should contain a single git commit hash which is the first commit without a SHA512 root commitment. +* `trusted-keys`: This file should contain a \n-delimited list of all PGP fingerprints of authorized commit signers (primary, not subkeys). +* `allow-revsig-commits`: This file should contain a \n-delimited list of git commit hashes. See next section for more info. + +Key expiry/revocation +--------------------- + +When a key (or subkey) which has signed old commits expires or is revoked, +verify-commits will start failing to verify all commits which were signed by +said key. In order to avoid bumping the root-of-trust `trusted-git-root` +file, individual commits which were signed by such a key can be added to the +`allow-revsig-commits` file. That way, the PGP signatures are still verified +but no new commits can be signed by any expired/revoked key. To easily build a +list of commits which need to be added, verify-commits.py can be edited to test +each commit with BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG set to both 1 and 0, and +those which need it set to 1 printed. diff --git a/contrib/verify-commits/allow-incorrect-sha512-commits b/contrib/verify-commits/allow-incorrect-sha512-commits new file mode 100644 index 0000000000..c572806f26 --- /dev/null +++ b/contrib/verify-commits/allow-incorrect-sha512-commits @@ -0,0 +1,2 @@ +f8feaa4636260b599294c7285bcf1c8b7737f74e +8040ae6fc576e9504186f2ae3ff2c8125de1095c diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index f0088cdca4..3abf82e529 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -102,3 +102,403 @@ bafd075c5e6a1088ef0f1aa0b0b224e026a3d3e0 c8d2473e6cb042e7275a10c49d3f6a4a91bf0166 386f4385ab04b0b2c3d47bddc0dc0f2de7354964 9f33dba05c01ecc5c56eb1284ab7d64d42f55171 +7466a26cab5d66665991433947964a638f5b957e +b43aba89e356ff95b706e80d4802f60fc46a569a +02b7e8319aef2a870264ad4fa2e3bb18664dcc36 +f686002a8eba820a40ac2f34a6e8f57b2b5cc54c +2b1c50b9352ab1dc40b0f877db23c1fa4048fae3 +2405ce1df043f778b8efb9205009500cbc17313a +4ad3b3c72c73d61e0a0cab541dca20acf651320d +4ba3d4f4393d81148422d24d222fe7ed00130194 +8ee5c7b747171e335793c74cd9d2f7491da58164 +872c921c0a208b04bd0713758e52fcab5b7c1684 +00d1680498c5550e7db1f359202d3433a092fafd +585db41e9ab7a6fb262c8bad7f427cdbdc497188 +18462960c0f13bd07d8f52b61e7d7bc17e991eea +0630974647dacaf25e7fcb7f9cbb785bb078ede6 +0f58d7f3d62f012f2584f5e781fc73de4763dd9e +3d16f581538b0974853e820508e8b3093269d2fd +66e91420ab233cf1dac64504e0dc129019bf8c0d +d8d9162f5bad39b2720dd2b2da237c6159e4755f +29fad97c320c892ab6a480c81e2078ec22ab354b +791c3ea61b4e49fd46a1a71b84ca99ddf69d2ff7 +a312e201ba56742499a5480b5f2115f01505c217 +ce56fdd2e8cdf94fd0ab76d71adbfa755e23ce7d +480f42630cbd598c04fa59ee0e406f56904ecffb +6012f1caf744ac9b53383d7d10a8f1b70ca2c0e1 +ded6a2afa549f693dcabb430ce0862f8631360c8 +07090c5339436f856e79a8036d1c85deeb453803 +0e265916d1c6a63e4a3821dab9db597b5ec64b46 +e4ffcacc2187d3419c8ea12b82fb06d82d8751d2 +e117cfe45eee9169409e74a44ef4a866be25bc35 +dcfe218626b05204e9fbc95ba5d95ca0eb72ec9b +23481fa50301201ef5a60675ef899aa6ce94ca03 +27c59dc502f29cf1d76290556c21e366145e3b2e +4a62ddd01873d18dbca96c81d756be1020249b45 +a233fb4f1d037e68ff70eef3a9f5b7bf1d631918 +b2089c51cc4af2f7e1c0ec75be9449ee222b1d69 +c997f8808256521397f1c003bb1e9896fee6eaa0 +5dc00f68c49c46a380a98d06233f90528b8e2557 +fe53d5f3636aed064823bc220d828c7ff08d1d52 +935eb8de039dec65669a96a1c3b86f4b03a1b86c +0277173b1defb63216d40a8d8805ae6d5d563c26 +2a30e67d20f76bbcd9a7d445f616f005316e0a1a +d32528e733f2711b34dbc41fbb2bb0f153bf7e9a +4cad91663df381d0dff8526f3b4aa74569dfb626 +1b06ed136f17b526360617a70026aed5ded5746c +895fbd768f0c89cea3f78acac58b233d4e3a145e +f0295becbf3ef1fb78095306408789253fe0c114 +8d573198638e52e2dbd9abc609861430f9d2bcc3 +9d9c4185fadaf243bb97c226e2fef16b65299699 +eebe4580bc8d6484d79ecb24dd87412221cf2ea7 +9cf6393a4f82b9c81d3b4b468a17a89db10531a2 +598a9c4e4dcd03c6d80fba005de729a6a3aeba7e +6970b30c6f1d2be7947295fe18f2390649b17a4b +f359afcc410432ed5d30001acda0c66741ee8935 +126000ba9e7ff16271be2f4eef3df99ade8d624f +b5e4b9b5100ec15217d43edb5f4149439f4b20a5 +b987ca4ee495a7fff82f0ac14ef0753bfb7586e2 +b03013396cb2f4bf25746388b3982a2c3616e16b +9a97f39afaa890caa7987c6bc001b9a66e3e74e8 +cad504bf4c302f7a72e0a0e191f3fdbafda7340f +45cf8a03cb57b8639a8d47323bde46ba22d9eeaf +b7450cdbd89a1c862f4d4d8bf093f8a0b5448f9c +0910cbe4ef31eb95fd76c7c2f820419fe64a3150 +92a810d04b906722c9efe60e3997243c71ff3d4c +45173fa6fca9537abb0a0554f731d14b9f89c456 +fd4ca17360e6fc0c9bb76bf6b5b07c9102c12728 +ddff3447f29b62d79a33f728791f42fa9436216e +36a5a4404836da323c755523fbd27563a8e84f94 +c991b304dee368f506cfee27ddaa333f1f82c518 +d38d1a3e75aa97ffa8755ddd431754a6d0942964 +a332a7d5a15214015f9553fdb2bcf80a1a4b8dc0 +604e08c83cf58ca7e7cda2ab284c1ace7bb12977 +18a1bbad98bd4321f15e7921d9aec91661499d90 +8049241e226c16bd07b029c0cb4b62ac40f0c923 +797441ee995aac59f55d59a93ecb55e8ecbe7dbc +62fdf9b07087b80d2142799bdd2324f61483359d +f60b4ad57912b78a96af08046a503f7905610a8c +13e31dd6548d64a5992f439e74bb424bf88aca04 +fbce66a982679b5409a295be5c99a2eef429cabf +9f2c2dba21855b8cb9b193b1819be73fa4a23a99 +a89221873a3ee2451c73b41bbe2d99d36f439d31 +3d6ad407770e13958e157bf026cae0bfb9254899 +901ba3e3819405306414628306746552b0aa1d28 +7a43fbb959c38e025e558e472ad57de357539894 +0d89fa0877930c6c8a539a656c1009ad8ab6755b +54aedc013744c86b11157423fa3cffc9a51eef02 +f0c1f8abb0182da557d07372b938f3a0a4bb906f +4ed818060ecf4a38a02c8cb48f6cbc78d2ee7708 +3bdf242fc68a8d767932c6214455d4d413effbc9 +5e468994fbb349e8eefc996954a31a67a34aaa15 +41aa9c4a801a01eca1fad22a7095372d23dace60 +2adbddb03840ad71e843c6c4a207a13e871cd1d4 +13e352dc53dec0127c5f94a60055d0ca829420dc +95e14dc81dd30ee0d396ad08dca9a6980d16eee1 +61fb80660f73e5aa5b69302ecc7ac33da206ba5a +05a761932edd05cf94ffe938908baf058f38632a +ee92243e66f2df03b3a759a8ffb75dc06f0cea0d +22cdf93c062eeaa0f8f9d6220f01b67240073dfb +76b33491596736ca804e3a29bd8398d7a1516ab7 +6e4e98ee8ce2da3cca2e2fd210e9e8dbc9b1c936 +c838283ecdfb9490425bb071b7c22e542de46c7c +5e3f5e4f25b65b583d3bfefac9e1148035781089 +f7388e93d3dd91a90239aedac4ec58404f103a2e +0a2f46b0158b6fc7244a585913b0925c0acf707f +dd561667cb7ccbbfed3134b05a565971ef6f5873 +6f01dcf63873a5e42798635ab4026c9a5f9fa213 +70fec9e36bcd1a3d93df019be084aaf89cecd7d7 +f9b74ef3fc74fd7d2aa94560820341f03cda8e12 +998c3046fab2b52bc9f141cfb588a18c05506a86 +89cc4f905e30b913ca20e4192d538cc5cbe2c38d +87d90efd69b64f769116956a5db89e536e9e3714 +5aeaa9ccd1568a77e075dbe2bd2435bd60c87c91 +bfb270acfa30713dc8c968bb9ee40cf5a2360359 +1b8c88451b0554502435d3883c528ad0aad1b09b +57ee73990f1ce29916adfd99f93eae1ccea1a43b +808c84f89d0edcef9ddaab0b849a382719f6ec9e +14b860bf64020451ced823b859da8cb912278ab9 +c63364610f4a041df1c1bd81d01b1f6856160749 +92eadc395071876d77f3babddc056b4325bdbabc +e93fff1463ae906fc986bf98c3b118c82f171546 +9ccafb1d7bdd172a9b963444072a844da379c4f7 +b4a509a3f817121c3df98ddfd96b2769e18a3e5a +dbc4ae03963014ab4b7957d62ba59dbd8f938c33 +8ddf60db7ad636b6a31b590251c671ded635fa1d +f199b8a33d9443a258a1f49a1a29674cd9ee9a20 +e542728cde676f218c552d841d0af29b92f9800b +763231051596b8e3455b839911ad6a3a1f1c3c74 +ff4cd6075b12fb32b9a906deea3ed033e3f9560a +9c3c9cdae3e20b5bdea91a0631edac5116bbc89f +93d20a734d2ee873832bed8ca5c05cf8e539c53c +ef8340d25f7c5dd5682bdecea97ce84cfce1493c +69c7ecef405d168f658a9cc7996da84c17f61e66 +4ce2f3d0d33346e9f0e96851689ee6550b2a72e3 +44e1fd926cfb0df0fbd8c41de8cd65ed8d5d6e18 +d6d2c8503c4039b682196d83a67dc28359c10c5c +ae233c4ec3d14a97c6195059f52873cdba2b4755 +0f399a9ff227896265cafab9b2e9fab6cdb9b5b9 +f4ed44ab4a8f9a87ba678d5fd1449fbf636103dc +7fcd61b2613c211bb042a82a889655178be6a212 +42973f834445d7735738bdba8847812ba3c34d95 +8df48b36ed3201d938b9974ecbee455d7dc2fb84 +96ac26e56627f0c24213fcd3a1cce9fc95f1f661 +cce94c518a46b7b0006f984bbe4d69e8749182d2 +801dd40666d1e6009920ad3ff755c7bb993b2a62 +ce829855cfca103dde55661fa1524e66b139d063 +b148803b181e30213e8a7f3bd89c8239e9dcb866 +c377feaad87f8109f85da6caf62602b30c20effc +b37cab65c63e051ebc5b491da9bd687581df94df +16e41844e7d6c5876d2caaeef6010656950c6ec5 +ee50c9e48786dea0d9df2e45805c25565c100fe3 +11dacc6154c42bc6fe3ba94c1823f8a46e4fe81a +791a0e6ddade27d1b69f4861a6640de60b9553cf +638e6c59da4fad987c437592174b188510193b2e +52f8877525d5238f3440e73710507be889d14127 +2a56baf395bf11835d784c4f8634f4525deed6a1 +bc561b4b7d6a3f71649d37d5eb9047c29efa2b13 +31809d6f8514c4a8d5677e947e3f1ebb0db210b9 +a31e9ad4f027955d43c04a05517244647e250161 +777519bd96f68c18150a0f5942f8f97a91937f5e +4eb1f39d421024d9666cec61deaf96715ffae4c6 +50fae68d416b4b8ec4ca192923dfd5ae9ea42773 +ce665863b137ac4a7470cf006a92aa7694faca71 +81f8c0378b2ab5ea0d7b65635cb529bd3c69127c +108222b9c323a05cc9339368f10ddd0859f62b43 +28f788e47e58f2b462351d6989348a4e1a241b2b +d81dccf191a48a6b59c3747d7b4ccbe3535dde40 +a90e6d2bffc422ddcdb771c53aac0bceb970a2c4 +91e49c51f1aecc9e1d75457f4920d52a4b0a133c +60dd9cc470584960431de425e2a9ffbed0e8034a +ede386c2193fc31351e193b3a8cf30030d6be62c +a084767b40c0d3ba8fa8f8d60f1e8d99a9dc3457 +3f726c99f819f97f2ab21b94d34c6b3129cd883a +77fc469fc78cdd87c29f398d46ac58dbb9ef62c0 +4ae6d0fbef60ccbecf8f23bb482e201b3678f7a3 +8858b6ddd3bce9daa08da6e05de3ca863a399c15 +22e301a3d56dc9e6878380ee92c7d19ca43119d2 +c484ec6c9b85ca4e331e395c564ae232fd0681dd +a46a671e253528e450bd57645c400bf761da07ab +655970d9c60ae6850daf452457e14e21047c0e1b +b6a48914c50631914192aa11b19205436a9c664d +7db65c363a0cc6ca7cdb04de9a973ab70013baad +6366941275344dac7e2130b0c972e90117d37ed0 +4fb2586661471a1572c2df2a5a091011d45eb7c4 +d7be7b39fa1021ec4518186afe145ee948e12a94 +85aec87b11ec41295558175c63f1f5a849460fdf +aeb31756276034dd506fdf97c8aaade0e7e584f5 +ac016e17d20253129a0287cee7e1d06b7ef15966 +bf74d377fb8e20140da6eac1407414928384bcea +2c811e08db651a4aed6ea0f7c1972d60de6de8ab +e5d26e47c7a482c072a7fe47bb84c56854734184 +96a63a3e0cefe920819bd42add0041837b1214a1 +e526ca6284b9e13be1b912b80dd73a34e739b539 +ecd21357f16106e541e9c2854ead2a906659b938 +4b5a7ce0c301ad971f383eb60f61bf9b4026efda +929fd7276c0f0c30b9416f61a6f5f35d763d81e4 +fa8a0639f7b0ce04030b72b4d5be4f0aa36fc5cb +f1f1605c22a6283bbfd757055fcf2b584a857709 +0c173a15ca1bf20999f74987988985508c9de463 +df0793f324e33066cc746c0cb1d053d35733d626 +2b0179d8a9b75397937126b36114df0dddeab40c +bf0a08be281dc42241e7f264c2a20515eb4781bb +3895e25a77363ae8b49358fb793f50fa8b271e2d +1fc783fc08bc078239537535f174ab8a489772c0 +1d4805ce04645f3203b0cfd3d66ea710e7433eb4 +d3b58704d1d325875fc605580c1c02b825c1bbcc +ed88e3194c4bc43aeafef929da7b419d03dea1ad +dd07f47b79628668e29cc0143b21e790100ee445 +65cc7aacfbfc7b747926375280a1d839e88d576b +080ec5209172ac9605f1434559dbb3c1e012b10a +416af3edf5b5ab265acf95568f2bc9eabd3d96de +e0a7801223fd573863939e76cb633f1dcc2d22c4 +4bc853b50fd9127687eb9e4f3b679dd261a4fa96 +c68a9a69278aa194fed96bd9733d32af3690a11e +c38f540298f0e188df5ed68fd56c623b9ac8331b +643fa0b22d70e459d7f7ec3d728ae4811dc5158f +e053e05c130549f43953f1d70e724dc9ce3e1b85 +75e898c094eea533d1dfaf141c6afccc3072c49f +2805d606bc46bf5589093a1b92d3542c13ce50c2 +32751807c9c06011eb689cba56b401a6302699c0 +30853e16d332816752dafcfca92147c7ffef5b54 +bea5b00cfe95cd37832305c0f93c339a22a7d79d +c871f323b418fac27bf834843ca26985010df53f +329fc1dce7a1c372c8b10c2f2f8732b2c60daff0 +1aefc94dd78d6e0c9209cb09fc16f53dedf42108 +8e5725666b519b61fcdc3141da5c6a57c1959909 +a4ca0b042365061020627a8c045cddacea3312ec +8bd16ee12fc8ef6723e0572c29b979c15b92b4f4 +87abe20fc118721cc5efdbd94a8462468cd1da2b +4b766fcdd4ca16399075d1e081a321b3b05ce516 +f6241b3e420e19f3f0507cbbc872fe9218916a02 +7ee523604851af62c0a47c07ee023a8710ef32f7 +776ba233e939fe41a74c6b2632b93a0679a32c71 +6a796b2b53fe542e0f340f250f4f20d69efed8d0 +23d78c4dd01bc74ba35db3e3df95280f6f1b2e22 +f4b15e2de97c4f8cdbb40bef4c9d0ab2807974d9 +fff72de5bf8ac7b70208e655f237b80e70e18851 +170bc2c381f86a523de2fc8b71d62ade66303c0d +314ebdfcb38d4b4c977579f787d5e1a20d068c94 +e9274839bf316b1972d80d28e45759f898edbf86 +75171f099e82e3527d7c3469b15891bd92227ec2 +3c5e6c94caf40395e031fbde44a0cca46fdd76ec +dc8fc0c73bebbc1c48ac5540026030c9cc00ec23 +492d22f92919d8d9d59568318c26c1e2ac4890cc +80c3a734298e824f9321c4efdd446086a3baad89 +47535d7c3ec79c5978cdcc03a5351ddbbb22538d +1b25b6df0f08f7474228c5b6ed13b58682e1e440 +c530c15180631cea95e9c292cf7fabde9dca9db3 +2723bcdce3248417e98e6c43207bef74d34076c1 +ed22eb4a62bd8d5369aaec87d4cbdc03c9f16368 +9111df9673beb6d6616d491a5478f09b5f14d040 +d86bb075bf6d1e78c1e4f3dd38b0ea828ef5ecfe +50a1cc0f0aef1514b917a5a3f4476967170b429d +6ce733747e160ca699711f2c47e686284ca9aa07 +b44adf92342ad4f9c343ba29c081a91687932936 +88799ea1b1c08f4bc1a487c9e3c2effd5e1650ae +080d7c700fc3291560d79fc590e05b8e2bad984f +12af74b289f8cdc6caf850dc6c802f9936b1e8b3 +8e4f7e72410df3ba430082c7cf385f26fd75b033 +8ac80412867118172dc4172494304e19969e9489 +f2734c2828f69d9cfd535e5eab0592a7674b2b61 +0b9fb682890b8fe10cec54072b809a5efe57d33d +5b029aaedb5fcf7cadd249607dd28eb3f233ab8c +79af9fbd8c3c0e54702a9c92b171f134bd4466c8 +c412fd805ddf3282dc2e1f28e30f51ffcb1f1da2 +111849345bb5140f86b48e730ceab4bff45fa2e9 +a0b1e57b20a17177ed5a9a54e4a8aab597a546b4 +ca209230c8e73745cf8cfc79f500c9c46e103306 +a230b0588788dbe1ac84622aea169c577b381241 +dfef6b6af08097f0676a2323085558fbbd3c48c6 +3192e5278abca7c1f3b4a2a7f77a0ce941c73985 +7c7ddd9ead99a8b5033a1a5d4698032c9e2b3a92 +10b930dde8f14e9cb661810e97a33bbf144fc55c +9225de2cf652fe2bf6e50636824cdb641546f57d +598ef9c44b3ea2cc142c175f077b493f39f5ba22 +c49355c7170a64bdd7864cc3ba9a64916b67fe7c +857d1e171e051b254a617f27b39f6a551054cee2 +21833f9456f6ad5bc06321ad6d9590f42ce0195c +8910b4717e5bb946ee6988f7fe9fd461f53a5935 +5703dff0939f05c7457cebd6fc61d88ab13afe41 +8bfa13b15b84cb372950fb7b25a1080173060b6a +ac23a7c1f19b3d8c326ffe75c8e13edf285f90fe +19be26afe3d04783a92d032b55bf3fb1e2ae63cc +f7ec7cfd38b543ba81ac7bed5b77f9a19739460b +36afd4db4442c45d4078b1a7ad16a1872b5bee0d +88c2ae3ed2bb5d367dd408c9255cd8f1e7a36c7d +a13a417cdcfdfd1f1b3bf997bb6ffe6e69b096b9 +d6064a89ac97dc0d2ce9da3982e1a4e25afaeda8 +7146d96de3e15a80cafbab2af48ff6f65d8e41bb +5628c70f2a44567695e5331fe2293c5b7f35b629 +7ff4a538a8682cdf02a4bcd6f15499c841001b73 +aa5fa642b0e7ce2ea55e2298886f212f11a8894e +8efd1c820b9a782d8608d54d924658536178295c +50a226563cd8d7c0a5e8448e87fede0eb72a8354 +b860915f8b0dae98e57a254d11575ea41f5c5a79 +d304fef3746039183f51b3ac8f4774dcf3a64f59 +53ab12d9318d5d195ccc77028b0e3ae66dc6e1fd +668de70be039a4f1ffcf20aeae2a22ee71fc55a8 +0fea960ca917b73aff853fe88476174c8a313863 +f89502306dcf6393a2c7b0efbb0fa728fc582137 +ff58b1c3bdff5e5f687f10f9e40ce495ca49674e +0b96abc35f1a9d46a27eeddd7df418d107c29c57 +b0b57a17306a7e963a4fe463f84e2b150a00a859 +4105cb6fd964ad13099ca83b1fdf3d35f3961f74 +23281a4dc3afc42a001346caec4dbb8193f0bb53 +8daf103fa138f9a184448ebf1c2e03b9dbd96f21 +02e5308c1b9f3771bbe49bc5036215fa2bd66aa9 +a65ced1a66575c652baf5084644b8647f531be8c +2456a835f0bc7796d9ff71f64837fa6790e2b7cc +9ec1330b455c1ab2eb6b89f8a2ab885677d4ae8a +0b738075bd43fbd4410e30a51e0498cbfd2b7513 +98c80e374b84e5a9c2d5c36889a0b1ebed5b814b +25720fc394e27a951bcad26095fb5a711bfacb8f +4cfd57d2e38207d78722ce8c9274ba8dd700d1cc +0fc1c31a878e93d938c67db3f958e82e3c39659f +df1ab5b4d67b46b5e9e840b1fbe0ff02520831f9 +5bc3b6cede8dabdf3f4f27ddb03723cbb7cde51a +c2ea1e6561caba3abffce361abc800822b9e0efe +caa2f106d704ec3ade63498031dd58d34510bc76 +dce853ef76ef90c46d84294225088d595467d08c +dbc8a8c86ae50059fddb2d6834fa5f0c9bbf9b71 +0f921e6a0492c4e9f037a9ed91f474885032d68c +041331e1da23e4136fd046ed870cdcc177464176 +e6ba5068f107ac234576e77cedbd748b665369c2 +76fcd9d5034143a5b041766552670d19f926097d +72bf1b3d0962304850a3ef5fe375db4bff1d0a39 +919db037f1f5cc73cdcaef92dd9cb0e7f5c8dec3 +c36229b0b2e9d4554053f5c9fc451ac29a493b1f +9e4bb312e6958d2baa309ba670e5eed1523c6f47 +d7ba4a233bd5a6f8fadee681c68a995e23fe36d7 +98514988a3d3e8b7dbf0463884a5c38f5ed5562d +5412c08c3cf13577566064edd04da021c37b7cbe +31bcc667863f368157efa1143a78623a5db8f0d1 +7bd1aa566fb4a4fe194f209085649f2c722b0cff +c4522e71c7e1d8ecfd70112e9375b9d00d6733a8 +e22f409f18881b63a8e747036584a71217f40e6e +97ec6e5c9098a1240655cfcab05b6cd5eedb6cd1 +bc121b0eb19713ec72002b5be03ba5ac35903a17 +c98f6b3d93a2cc1b49a6db425ea2b661089d0f9e +0de7fd36de57a68e543b4c1f184fba192c398c73 +e662d281b837c25b2b70525aa8fe8af894339823 +44adf683ad232db8ce0cb89b3e236a1f5944cfb0 +cb2ed300a89ebf9f0654da869ced665ed8b2abe7 +0a6d48d9ed60b0b02177059ab116f8f46d2cbed3 +b42291334651fff46dbfe5947a726f65cb9d7dfe +e5364991daecb73aca3bb5ac37f2619d7a89211b +4a2b170c075ce703cbdc82519a48016a9ee3f99c +924de0bd75a7f75df65d7d15f9d1587a2e794abf +1253f8692fc3a11be9430685cd405236a68df6c3 +2b799ae9e1e0a540f9a5971ddf27d83254668279 +c9bdf9a75f9fde8cd011e4aa94be4ed4347078a3 +3d69ecb4edeb80003a1a41442e320898a30dbd9c +f08222e882b18c1f279308636e03beceece2dbf1 +23e03f8d26d7bd03273a5dcbdcfe3905dfb49ffb +03dd707dc027fbf6f24120213f8eb66571600374 +d0754799698de2c032abcb8198ee5d5401063213 +072116fceb2294b97d1c40f79305f2e3ff71812b +e66cc1d58e16bf1650dd6479fed64ecaca8c6098 +f137753a2dcd8229f89d1d1ac28039364e5850b4 +61d191fbf953700ba8aeadc9c8cf4c195efbd10c +76f3c02fb01a6df98fbd8c16ac21d159d4649d37 +6013c73b3312e11b447ed387426749014716f820 +6faffb8a83db3f209a303a4464dbdd597faad5a4 +cc9e8aca5f950c78dcfeff63c441ba993c1fe12f +8ca69a2a88a77eb06149fa049ab1a7e6de38b321 +2f71490d21796594ca6f55e375558944de9db5a0 +08cc5fd666456cb476467473ed1880c90c92dedb +e31a43c725ebe641d7c219c3886eee18eebf0bb8 +52b5a8785de760a204b2b0aab19dfaf79c2c3ff0 +483e8e4f4875a1a621ec9e9df2880d3037d95ed7 +1e5799c52535a3fc20e885916f1e7ed33ecc7f46 +a82e5d8220bbc8b5d786bed99b0876f530b9b7cc +7fe6c5c993706e8395cdaf7977bee793c06f48f3 +2a0836f6d5e7c1d7e97bedb0e0ea33dcaf981f77 +ddc308068d69c6c9aa629ee3c4ce75e1d1cf08b5 +ec139a5621a9c9f03e1988391a3c7c6c5d849776 +c01a6c48b982d625fd9f4f69005878781d3d56fa +95a983d56dbda457e3bf8766d59bac74c7aa5699 +760741a00833876976389ed7a6b73f36ee5b4c13 +6e5e5abba6f8bbbe61c22795df440dfafcfdc378 +cf2cecb18779ce83de9adebf382dff1c19b12840 +af9b7a9f2f73b1a2f9728106774dd13e8d1cdd8d +115735d547fdeade822f547eb3e8c8f9961a9b07 +c2c69edf37b5c02aafa01d0407dadbf5ef8751b5 +a072d1a83787e786d074a4b5871b0b961781f7c6 +ed2cd59e258f756b2eaed7909a60956ade6ef7ee +ae5575ba41c8a782805afb1c08730343cfc22397 +6ff2c8d29f6b5a5c2ce63f0a16f3bb0dbd049451 +a80de15113166354cdf208e3d8b6e25f4511a591 +06bd4f637f15e769f088d9051a5af94bbb0217a3 +6700cc993cc07fb0f5b8b577ff8c4afcf0b18274 +37f9a1f627c0995d89b62923e75cd092600894f9 +8844ef15ded02d5ed86fb95aaf251235fcef2396 +1b87e5b5b184a0a6c683eda23b36393822b57f03 +e2bf830bb6c1bfa038c943dd6f5d92a406bd723f +423ca302a3ee87000530da3c105f269b8fabece7 +4e14afe42fdd468d5de11df8cc13defdcb8e83f8 +3e90fe6534206412ea22beaa445cf20d28fbe718 +88b77c7da0a672c89e24df37ea6e9085b4e2a05c +0ad104190465d8d65c2344bbe10dcf3df025d86c +5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a +59ad56851a342d2c62f6b38bf15002b23ab439e1 diff --git a/contrib/verify-commits/allow-unclean-merge-commits b/contrib/verify-commits/allow-unclean-merge-commits new file mode 100644 index 0000000000..7aab274b9a --- /dev/null +++ b/contrib/verify-commits/allow-unclean-merge-commits @@ -0,0 +1,4 @@ +6052d509105790a26b3ad5df43dd61e7f1b24a12 +3798e5de334c3deb5f71302b782f6b8fbd5087f1 +326ffed09bfcc209a2efd6a2ebc69edf6bd200b5 +97d83739db0631be5d4ba86af3616014652c00ec diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 8f3e4b8063..7a10ba7d7d 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C INPUT=$(cat /dev/stdin) VALID=false REVSIG=false @@ -57,7 +58,7 @@ if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "^\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index c21febb9e9..4db4a90853 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,8 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) 2014-2015 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then exit 0 fi @@ -12,9 +13,9 @@ while read LINE; do if [ "$4" != "refs/heads/master" ]; then continue fi - if ! ./contrib/verify-commits/verify-commits.sh $3 > /dev/null 2>&1; then + if ! ./contrib/verify-commits/verify-commits.py $3 > /dev/null 2>&1; then echo "ERROR: A commit is not signed, can't push" - ./contrib/verify-commits/verify-commits.sh + ./contrib/verify-commits/verify-commits.py exit 1 fi done < /dev/stdin diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index e560b98d02..c60f8ab695 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -11049f4fe62606d1b0380a9ef800ac130f0fbadf +82bcf405f6db1d55b684a1f63a4aabad376cdad7 diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py new file mode 100755 index 0000000000..a9e4977715 --- /dev/null +++ b/contrib/verify-commits/verify-commits.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Verify commits against a trusted keys list.""" +import argparse +import hashlib +import os +import subprocess +import sys +import time + +GIT = os.getenv('GIT', 'git') + +def tree_sha512sum(commit='HEAD'): + """Calculate the Tree-sha512 for the commit. + + This is copied from github-merge.py.""" + + # request metadata for entire tree, recursively + files = [] + blob_by_name = {} + for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): + name_sep = line.index(b'\t') + metadata = line[:name_sep].split() # perms, 'blob', blobid + assert metadata[1] == b'blob' + name = line[name_sep + 1:] + files.append(name) + blob_by_name[name] = metadata[2] + + files.sort() + # open connection to git-cat-file in batch mode to request data for all blobs + # this is much faster than launching it per file + p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + overall = hashlib.sha512() + for f in files: + blob = blob_by_name[f] + # request blob + p.stdin.write(blob + b'\n') + p.stdin.flush() + # read header: blob, "blob", size + reply = p.stdout.readline().split() + assert reply[0] == blob and reply[1] == b'blob' + size = int(reply[2]) + # hash the blob data + intern = hashlib.sha512() + ptr = 0 + while ptr < size: + bs = min(65536, size - ptr) + piece = p.stdout.read(bs) + if len(piece) == bs: + intern.update(piece) + else: + raise IOError('Premature EOF reading git cat-file output') + ptr += bs + dig = intern.hexdigest() + assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data + # update overall hash with file hash + overall.update(dig.encode("utf-8")) + overall.update(" ".encode("utf-8")) + overall.update(f) + overall.update("\n".encode("utf-8")) + p.stdin.close() + if p.wait(): + raise IOError('Non-zero return value executing git cat-file') + return overall.hexdigest() + +def main(): + # Parse arguments + parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]') + parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check') + parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER') + parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>') + args = parser.parse_args() + + # get directory of this program and read data files + dirname = os.path.dirname(os.path.abspath(__file__)) + print("Using verify-commits data from " + dirname) + verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] + verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] + revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() + unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() + incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() + + # Set commit and branch and set variables + current_commit = args.commit + if ' ' in current_commit: + print("Commit must not contain spaces", file=sys.stderr) + sys.exit(1) + verify_tree = args.verify_tree + no_sha1 = True + prev_commit = "" + initial_commit = current_commit + branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True).splitlines()[0] + + # Iterate through commits + while True: + if current_commit == verified_root: + print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) + sys.exit(0) + if current_commit == verified_sha512_root: + if verify_tree: + print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr) + verify_tree = False + no_sha1 = False + + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1" + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0" + + # Check that the commit (and parents) was signed with a trusted key + if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL): + if prev_commit != "": + print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) + print("Parents are:", file=sys.stderr) + parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True).splitlines()[0].split(' ') + for parent in parents: + subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr) + else: + print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr) + sys.exit(1) + + # Check the Tree-SHA512 + if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed: + tree_hash = tree_sha512sum(current_commit) + if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True).splitlines(): + print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr) + sys.exit(1) + + # Merge commits should only have two parents + parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True).splitlines()[0].split(' ') + if len(parents) > 2: + print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr) + sys.exit(1) + + # Check that the merge commit is clean + commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True).splitlines()[0]) + check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days + allow_unclean = current_commit in unclean_merge_allowed + if len(parents) == 2 and check_merge and not allow_unclean: + current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True).splitlines()[0] + subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) + subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL) + recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True).splitlines()[0] + if current_tree != recreated_tree: + print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) + subprocess.call([GIT, 'diff', current_commit]) + subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + sys.exit(1) + subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + + prev_commit = current_commit + current_commit = parents[0] + +if __name__ == '__main__': + main() diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh deleted file mode 100755 index 532b97a438..0000000000 --- a/contrib/verify-commits/verify-commits.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/sh -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -DIR=$(dirname "$0") -[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0") - -echo "Using verify-commits data from ${DIR}" - -VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") -VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit") -REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") - -HAVE_GNU_SHA512=1 -[ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0 - -if [ x"$1" = "x" ]; then - CURRENT_COMMIT="HEAD" -else - CURRENT_COMMIT="$1" -fi - -if [ "${CURRENT_COMMIT#* }" != "$CURRENT_COMMIT" ]; then - echo "Commit must not contain spaces?" > /dev/stderr - exit 1 -fi - -VERIFY_TREE=0 -if [ x"$2" = "x--tree-checks" ]; then - VERIFY_TREE=1 -fi - -NO_SHA1=1 -PREV_COMMIT="" -INITIAL_COMMIT="${CURRENT_COMMIT}" - -while true; do - if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then - echo "There is a valid path from \"$INITIAL_COMMIT\" to $VERIFIED_ROOT where all commits are signed!" - exit 0 - fi - - if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then - if [ "$VERIFY_TREE" = "1" ]; then - echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr - fi - VERIFY_TREE=0 - NO_SHA1=0 - fi - - if [ "$NO_SHA1" = "1" ]; then - export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0 - else - export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1 - fi - - if [ "${REVSIG_ALLOWED#*$CURRENT_COMMIT}" != "$REVSIG_ALLOWED" ]; then - export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 - else - export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 - fi - - if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit "$CURRENT_COMMIT" > /dev/null; then - if [ "$PREV_COMMIT" != "" ]; then - echo "No parent of $PREV_COMMIT was signed with a trusted key!" > /dev/stderr - echo "Parents are:" > /dev/stderr - PARENTS=$(git show -s --format=format:%P $PREV_COMMIT) - for PARENT in $PARENTS; do - git show -s $PARENT > /dev/stderr - done - else - echo "$CURRENT_COMMIT was not signed with a trusted key!" > /dev/stderr - fi - exit 1 - fi - - # We always verify the top of the tree - if [ "$VERIFY_TREE" = 1 -o "$PREV_COMMIT" = "" ]; then - IFS_CACHE="$IFS" - IFS=' -' - for LINE in $(git ls-tree --full-tree -r "$CURRENT_COMMIT"); do - case "$LINE" in - "12"*) - echo "Repo contains symlinks" > /dev/stderr - IFS="$IFS_CACHE" - exit 1 - ;; - esac - done - IFS="$IFS_CACHE" - - FILE_HASHES="" - for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do - if [ "$HAVE_GNU_SHA512" = 1 ]; then - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST _; echo $FIRST; } ) - else - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST _; echo $FIRST; } ) - fi - [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"' -' - FILE_HASHES="$FILE_HASHES$HASH $FILE" - done - - if [ "$HAVE_GNU_SHA512" = 1 ]; then - TREE_HASH="$(echo "$FILE_HASHES" | sha512sum)" - else - TREE_HASH="$(echo "$FILE_HASHES" | shasum -a 512)" - fi - HASH_MATCHES=0 - MSG="$(git show -s --format=format:%B "$CURRENT_COMMIT" | tail -n1)" - - case "$MSG -" in - "Tree-SHA512: $TREE_HASH") - HASH_MATCHES=1;; - esac - - if [ "$HASH_MATCHES" = "0" ]; then - echo "Tree-SHA512 did not match for commit $CURRENT_COMMIT" > /dev/stderr - exit 1 - fi - fi - - PARENTS=$(git show -s --format=format:%P "$CURRENT_COMMIT") - for PARENT in $PARENTS; do - PREV_COMMIT="$CURRENT_COMMIT" - CURRENT_COMMIT="$PARENT" - break - done -done diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index e0266bf08a..fc7492ad3b 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,7 @@ ### The script returns 0 if everything passes the checks. It returns 1 if either the ### signature check or the hash check doesn't pass. If an error occurs the return value is 2 +export LC_ALL=C function clean_up { for file in $* do diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index bf4978d143..15f8108cf0 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C if [ -z "$OSSLSIGNCODE" ]; then OSSLSIGNCODE=osslsigncode fi diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 60768dc59a..fa9e669308 100644 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -30,7 +30,7 @@ import signal import struct import sys -if not (sys.version_info.major >= 3 and sys.version_info.minor >= 5): +if (sys.version_info.major, sys.version_info.minor) < (3, 5): print("This example only works with Python 3.5 and greater") sys.exit(1) diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py index 0df843c9a3..d05ecc2623 100644 --- a/contrib/zmq/zmq_sub3.4.py +++ b/contrib/zmq/zmq_sub3.4.py @@ -34,7 +34,7 @@ import signal import struct import sys -if not (sys.version_info.major >= 3 and sys.version_info.minor >= 4): +if (sys.version_info.major, sys.version_info.minor) < (3, 4): print("This example only works with Python 3.4 and greater") sys.exit(1) |