diff options
Diffstat (limited to 'contrib/devtools')
| -rw-r--r-- | contrib/devtools/README.md | 83 | ||||
| -rwxr-xr-x | contrib/devtools/check-doc.py | 45 | ||||
| -rwxr-xr-x | contrib/devtools/clang-format-diff.py | 164 | ||||
| -rwxr-xr-x | contrib/devtools/fix-copyright-headers.py | 69 | ||||
| -rwxr-xr-x | contrib/devtools/github-merge.py | 235 | ||||
| -rwxr-xr-x | contrib/devtools/github-merge.sh | 185 | ||||
| -rwxr-xr-x | contrib/devtools/security-check.py | 4 | ||||
| -rwxr-xr-x | contrib/devtools/symbol-check.py | 5 | ||||
| -rwxr-xr-x | contrib/devtools/update-translations.py | 22 |
9 files changed, 550 insertions, 262 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 2e70c5adcc..1103ca86c5 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -2,19 +2,68 @@ Contents ======== This directory contains tools for developers working on this repository. +check-doc.py +============ + +Check if all command line args are documented. The return value indicates the +number of undocumented args. + clang-format.py =============== A script to format cpp source code according to [.clang-format](../../src/.clang-format). This should only be applied to new files or files which are currently not actively developed on. Also, git subtrees are not subject to formatting. -github-merge.sh +clang-format-diff.py +=================== + +A script to format unified git diffs according to [.clang-format](../../src/.clang-format). + +For instance, to format the last commit with 0 lines of context, +the script should be called from the git root folder as follows. + +``` +git diff -U0 HEAD~1.. | ./contrib/devtools/clang-format-diff.py -p1 -i -v +``` + +fix-copyright-headers.py +======================== + +Every year newly updated files need to have its copyright headers updated to reflect the current year. +If you run this script from the root folder it will automatically update the year on the copyright header for all +source files if these have a git commit from the current year. + +For example a file changed in 2015 (with 2015 being the current year): + +```// Copyright (c) 2009-2013 The Bitcoin Core developers``` + +would be changed to: + +```// Copyright (c) 2009-2015 The Bitcoin Core developers``` + +git-subtree-check.sh +==================== + +Run this script from the root of the repository to verify that a subtree matches the contents of +the commit it claims to have been updated to. + +To use, make sure that you have fetched the upstream repository branch in which the subtree is +maintained: +* for `src/secp256k1`: https://github.com/bitcoin/secp256k1.git (branch master) +* for `src/leveldb`: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork) +* for `src/univalue`: https://github.com/bitcoin/univalue.git (branch master) + +Usage: `git-subtree-check.sh DIR COMMIT` + +`COMMIT` may be omitted, in which case `HEAD` is used. + +github-merge.py =============== A small script to automate merging pull-requests securely and sign them with GPG. For example: - ./github-merge.sh bitcoin/bitcoin 3077 + ./github-merge.py 3077 (in any git repository) will help you merge pull request #3077 for the bitcoin/bitcoin repository. @@ -41,21 +90,6 @@ Configuring the github-merge tool for the bitcoin repository is done in the foll git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) git config --global user.signingkey mykeyid (if you want to GPG sign) -fix-copyright-headers.py -======================== - -Every year newly updated files need to have its copyright headers updated to reflect the current year. -If you run this script from src/ it will automatically update the year on the copyright header for all -.cpp and .h files if these have a git commit from the current year. - -For example a file changed in 2014 (with 2014 being the current year): - -```// Copyright (c) 2009-2013 The Bitcoin Core developers``` - -would be changed to: - -```// Copyright (c) 2009-2014 The Bitcoin Core developers``` - optimize-pngs.py ================ @@ -98,18 +132,3 @@ It will do the following automatically: - add missing translations to the build system (TODO) See doc/translation-process.md for more information. - -git-subtree-check.sh -==================== - -Run this script from the root of the repository to verify that a subtree matches the contents of -the commit it claims to have been updated to. - -To use, make sure that you have fetched the upstream repository branch in which the subtree is -maintained: -* for src/secp256k1: https://github.com/bitcoin/secp256k1.git (branch master) -* for sec/leveldb: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork) - -Usage: git-subtree-check.sh DIR COMMIT - -COMMIT may be omitted, in which case HEAD is used. diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py new file mode 100755 index 0000000000..8c73cf1e8a --- /dev/null +++ b/contrib/devtools/check-doc.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +''' +This checks if all command line args are documented. +Return value is 0 to indicate no error. + +Author: @MarcoFalke +''' + +from subprocess import check_output +import re + +FOLDER_GREP = 'src' +FOLDER_TEST = 'src/test/' +CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/%s' % FOLDER_GREP +CMD_GREP_ARGS = r"egrep -r -I '(map(Multi)?Args(\.count\(|\[)|Get(Bool)?Arg\()\"\-[^\"]+?\"' %s | grep -v '%s'" % (CMD_ROOT_DIR, FOLDER_TEST) +CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' %s" % (CMD_ROOT_DIR) +REGEX_ARG = re.compile(r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"') +REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")') +# list unsupported, deprecated and duplicate args as they need no documentation +SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay']) + +def main(): + used = check_output(CMD_GREP_ARGS, shell=True) + docd = check_output(CMD_GREP_DOCS, shell=True) + + args_used = set(re.findall(REGEX_ARG,used)) + args_docd = set(re.findall(REGEX_DOC,docd)).union(SET_DOC_OPTIONAL) + args_need_doc = args_used.difference(args_docd) + args_unknown = args_docd.difference(args_used) + + print "Args used : %s" % len(args_used) + print "Args documented : %s" % len(args_docd) + print "Args undocumented: %s" % len(args_need_doc) + print args_need_doc + print "Args unknown : %s" % len(args_unknown) + print args_unknown + + exit(len(args_need_doc)) + +if __name__ == "__main__": + main() diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py new file mode 100755 index 0000000000..13d2573b9f --- /dev/null +++ b/contrib/devtools/clang-format-diff.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# +#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. +# +# ============================================================ +# +# University of Illinois/NCSA +# Open Source License +# +# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign. +# All rights reserved. +# +# Developed by: +# +# LLVM Team +# +# University of Illinois at Urbana-Champaign +# +# http://llvm.org +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal with +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimers. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimers in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of the LLVM Team, University of Illinois at +# Urbana-Champaign, nor the names of its contributors may be used to +# endorse or promote products derived from this Software without specific +# prior written permission. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +# SOFTWARE. +# +# ============================================================ +# +#===------------------------------------------------------------------------===# + +r""" +ClangFormat Diff Reformatter +============================ + +This script reads input from a unified diff and reformats all the changed +lines. This is useful to reformat all the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 HEAD^ | clang-format-diff.py -p1 -i + svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i + +""" + +import argparse +import difflib +import re +import string +import subprocess +import StringIO +import sys + + +# Change this to the full path if clang-format is not on the path. +binary = 'clang-format' + + +def main(): + parser = argparse.ArgumentParser(description= + 'Reformat changed lines in diff. Without -i ' + 'option just output the diff that would be ' + 'introduced.') + parser.add_argument('-i', action='store_true', default=False, + help='apply edits to files instead of displaying a diff') + parser.add_argument('-p', metavar='NUM', default=0, + help='strip the smallest prefix containing P slashes') + parser.add_argument('-regex', metavar='PATTERN', default=None, + help='custom pattern selecting file paths to reformat ' + '(case sensitive, overrides -iregex)') + parser.add_argument('-iregex', metavar='PATTERN', default= + r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto' + r'|protodevel|java)', + help='custom pattern selecting file paths to reformat ' + '(case insensitive, overridden by -regex)') + parser.add_argument('-sort-includes', action='store_true', default=False, + help='let clang-format sort include blocks') + parser.add_argument('-v', '--verbose', action='store_true', + help='be more verbose, ineffective without -i') + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) + if match: + filename = match.group(2) + if filename == None: + continue + + if args.regex is not None: + if not re.match('^%s$' % args.regex, filename): + continue + else: + if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search('^@@.*\+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1; + lines_by_file.setdefault(filename, []).extend( + ['-lines', str(start_line) + ':' + str(end_line)]) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.iteritems(): + if args.i and args.verbose: + print 'Formatting', filename + command = [binary, filename] + if args.i: + command.append('-i') + if args.sort_includes: + command.append('-sort-includes') + command.extend(lines) + command.extend(['-style=file', '-fallback-style=none']) + p = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=None, stdin=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode); + + if not args.i: + with open(filename) as f: + code = f.readlines() + formatted_code = StringIO.StringIO(stdout).readlines() + diff = difflib.unified_diff(code, formatted_code, + filename, filename, + '(before formatting)', '(after formatting)') + diff_string = string.join(diff, '') + if len(diff_string) > 0: + sys.stdout.write(diff_string) + +if __name__ == '__main__': + main() diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py index 5e84952548..b6414a551f 100755 --- a/contrib/devtools/fix-copyright-headers.py +++ b/contrib/devtools/fix-copyright-headers.py @@ -1,53 +1,46 @@ #!/usr/bin/env python ''' -Run this script inside of src/ and it will look for all the files -that were changed this year that still have the last year in the -copyright headers, and it will fix the headers on that file using -a perl regex one liner. +Run this script to update all the copyright headers of files +that were changed this year. -For example: if it finds something like this and we're in 2014 +For example: -// Copyright (c) 2009-2013 The Bitcoin Core developers +// Copyright (c) 2009-2012 The Bitcoin Core developers it will change it to -// Copyright (c) 2009-2014 The Bitcoin Core developers - -It will do this for all the files in the folder and its children. - -Author: @gubatron +// Copyright (c) 2009-2015 The Bitcoin Core developers ''' import os import time +import re year = time.gmtime()[0] -last_year = year - 1 -command = "perl -pi -e 's/%s The Bitcoin/%s The Bitcoin/' %s" -listFilesCommand = "find . | grep %s" - -extensions = [".cpp",".h"] - -def getLastGitModifiedDate(filePath): - gitGetLastCommitDateCommand = "git log " + filePath +" | grep Date | head -n 1" - p = os.popen(gitGetLastCommitDateCommand) - result = "" - for l in p: - result = l - break - result = result.replace("\n","") - return result +CMD_GIT_DATE = 'git log --format=@%%at -1 %s | date +"%%Y" -u -f -' +CMD_REGEX= "perl -pi -e 's/(20\d\d)(?:-20\d\d)? The Bitcoin/$1-%s The Bitcoin/' %s" +REGEX_CURRENT= re.compile("%s The Bitcoin" % year) +CMD_LIST_FILES= "find %s | grep %s" -n=1 -for extension in extensions: - foundFiles = os.popen(listFilesCommand % extension) - for filePath in foundFiles: - filePath = filePath[1:-1] - if filePath.endswith(extension): - filePath = os.getcwd() + filePath - modifiedTime = getLastGitModifiedDate(filePath) - if len(modifiedTime) > 0 and str(year) in modifiedTime: - print n,"Last Git Modified: ", modifiedTime, " - ", filePath - os.popen(command % (last_year,year,filePath)) - n = n + 1 +FOLDERS = ["./qa", "./src"] +EXTENSIONS = [".cpp",".h", ".py"] +def get_git_date(file_path): + r = os.popen(CMD_GIT_DATE % file_path) + for l in r: + # Result is one line, so just return + return l.replace("\n","") + return "" +n=1 +for folder in FOLDERS: + for extension in EXTENSIONS: + for file_path in os.popen(CMD_LIST_FILES % (folder, extension)): + file_path = os.getcwd() + file_path[1:-1] + if file_path.endswith(extension): + git_date = get_git_date(file_path) + if str(year) == git_date: + # Only update if current year is not found + if REGEX_CURRENT.search(open(file_path, "r").read()) is None: + print n,"Last git edit", git_date, "-", file_path + os.popen(CMD_REGEX % (year,file_path)) + n = n + 1 diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py new file mode 100755 index 0000000000..c8dcaae268 --- /dev/null +++ b/contrib/devtools/github-merge.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python2 +# Copyright (c) 2016 Bitcoin Core Developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# This script will locally construct a merge commit for a pull request on a +# github repository, inspect it, sign it and optionally push it. + +# The following temporary branches are created/overwritten and deleted: +# * pull/$PULL/base (the current master we're merging onto) +# * pull/$PULL/head (the current state of the remote pull request) +# * pull/$PULL/merge (github's merge) +# * pull/$PULL/local-merge (our merge) + +# In case of a clean merge that is accepted by the user, the local branch with +# name $BRANCH is overwritten with the merged result, and optionally pushed. +from __future__ import division,print_function,unicode_literals +import os,sys +from sys import stdin,stdout,stderr +import argparse +import subprocess + +# External tools (can be overridden using environment) +GIT = os.getenv('GIT','git') +BASH = os.getenv('BASH','bash') + +# OS specific configuration for terminal attributes +ATTR_RESET = '' +ATTR_PR = '' +COMMIT_FORMAT = '%h %s (%an)%d' +if os.name == 'posix': # if posix, assume we can use basic terminal escapes + ATTR_RESET = '\033[0m' + ATTR_PR = '\033[1;36m' + COMMIT_FORMAT = '%C(bold blue)%h%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset' + +def git_config_get(option, default=None): + ''' + Get named configuration option from git repository. + ''' + try: + return subprocess.check_output([GIT,'config','--get',option]).rstrip() + except subprocess.CalledProcessError as e: + return default + +def retrieve_pr_title(repo,pull): + ''' + Retrieve pull request title from github. + Return None if no title can be found, or an error happens. + ''' + import urllib2,json + try: + req = urllib2.Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + result = urllib2.urlopen(req) + result = json.load(result) + return result['title'] + except Exception as e: + print('Warning: unable to retrieve pull title from github: %s' % e) + return None + +def ask_prompt(text): + print(text,end=" ",file=stderr) + reply = stdin.readline().rstrip() + print("",file=stderr) + return reply + +def parse_arguments(branch): + epilog = ''' + In addition, you can set the following git configuration variables: + githubmerge.repository (mandatory), + user.signingkey (mandatory), + githubmerge.host (default: git@github.com), + githubmerge.branch (default: master), + githubmerge.testcmd (default: none). + ''' + parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', + epilog=epilog) + parser.add_argument('pull', metavar='PULL', type=int, nargs=1, + help='Pull request ID to merge') + parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', + default=branch, help='Branch to merge against (default: '+branch+')') + return parser.parse_args() + +def main(): + # Extract settings from git repo + repo = git_config_get('githubmerge.repository') + host = git_config_get('githubmerge.host','git@github.com') + branch = git_config_get('githubmerge.branch','master') + testcmd = git_config_get('githubmerge.testcmd') + signingkey = git_config_get('user.signingkey') + if repo is None: + print("ERROR: No repository configured. Use this command to set:", file=stderr) + print("git config githubmerge.repository <owner>/<repo>", file=stderr) + exit(1) + if signingkey is None: + print("ERROR: No GPG signing key set. Set one using:",file=stderr) + print("git config --global user.signingkey <key>",file=stderr) + exit(1) + + host_repo = host+":"+repo # shortcut for push/pull target + + # Extract settings from command line + args = parse_arguments(branch) + pull = str(args.pull[0]) + branch = args.branch + + # Initialize source branches + head_branch = 'pull/'+pull+'/head' + base_branch = 'pull/'+pull+'/base' + merge_branch = 'pull/'+pull+'/merge' + local_merge_branch = 'pull/'+pull+'/local-merge' + + devnull = open(os.devnull,'w') + try: + subprocess.check_call([GIT,'checkout','-q',branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot check out branch %s." % (branch), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*']) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/heads/'+branch+':refs/heads/'+base_branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find branch %s on %s." % (branch,host_repo), file=stderr) + exit(3) + subprocess.check_call([GIT,'checkout','-q',base_branch]) + subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) + subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch]) + + try: + # Create unsigned merge commit. + title = retrieve_pr_title(repo,pull) + if title: + firstline = 'Merge #%s: %s' % (pull,title) + else: + firstline = 'Merge #%s' % (pull,) + message = firstline + '\n\n' + message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') + try: + subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot be merged cleanly.",file=stderr) + subprocess.check_call([GIT,'merge','--abort']) + exit(4) + logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8') + if logmsg.rstrip() != firstline.rstrip(): + print("ERROR: Creating merge failed (already merged?).",file=stderr) + exit(4) + + print('%s#%s%s %s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title)) + subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + print() + # Run test command if configured. + if testcmd: + # Go up to the repository's root. + toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip() + os.chdir(toplevel) + if subprocess.call(testcmd,shell=True): + print("ERROR: Running %s failed." % testcmd,file=stderr) + exit(5) + + # Show the created merge. + diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch]) + subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch]) + if diff: + print("WARNING: merge differs from github!",file=stderr) + reply = ask_prompt("Type 'ignore' to continue.") + if reply.lower() == 'ignore': + print("Difference with github ignored.",file=stderr) + else: + exit(6) + reply = ask_prompt("Press 'd' to accept the diff.") + if reply.lower() == 'd': + print("Diff accepted.",file=stderr) + else: + print("ERROR: Diff rejected.",file=stderr) + exit(6) + else: + # Verify the result manually. + print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) + print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr) + print("Type 'exit' when done.",file=stderr) + if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt + os.putenv('debian_chroot',pull) + subprocess.call([BASH,'-i']) + reply = ask_prompt("Type 'm' to accept the merge.") + if reply.lower() == 'm': + print("Merge accepted.",file=stderr) + else: + print("ERROR: Merge rejected.",file=stderr) + exit(7) + + # Sign the merge commit. + reply = ask_prompt("Type 's' to sign off on the merge.") + if reply == 's': + try: + subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + except subprocess.CalledProcessError as e: + print("Error signing, exiting.",file=stderr) + exit(1) + else: + print("Not signing off on merge, exiting.",file=stderr) + exit(1) + + # Put the result in branch. + subprocess.check_call([GIT,'checkout','-q',branch]) + subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch]) + finally: + # Clean up temporary branches. + subprocess.call([GIT,'checkout','-q',branch]) + subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) + + # Push the result. + reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch)) + if reply.lower() == 'push': + subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + +if __name__ == '__main__': + main() + diff --git a/contrib/devtools/github-merge.sh b/contrib/devtools/github-merge.sh deleted file mode 100755 index afb53f0390..0000000000 --- a/contrib/devtools/github-merge.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash - -# This script will locally construct a merge commit for a pull request on a -# github repository, inspect it, sign it and optionally push it. - -# The following temporary branches are created/overwritten and deleted: -# * pull/$PULL/base (the current master we're merging onto) -# * pull/$PULL/head (the current state of the remote pull request) -# * pull/$PULL/merge (github's merge) -# * pull/$PULL/local-merge (our merge) - -# In case of a clean merge that is accepted by the user, the local branch with -# name $BRANCH is overwritten with the merged result, and optionally pushed. - -REPO="$(git config --get githubmerge.repository)" -if [[ "d$REPO" == "d" ]]; then - echo "ERROR: No repository configured. Use this command to set:" >&2 - echo "git config githubmerge.repository <owner>/<repo>" >&2 - echo "In addition, you can set the following variables:" >&2 - echo "- githubmerge.host (default git@github.com)" >&2 - echo "- githubmerge.branch (default master)" >&2 - echo "- githubmerge.testcmd (default none)" >&2 - exit 1 -fi - -HOST="$(git config --get githubmerge.host)" -if [[ "d$HOST" == "d" ]]; then - HOST="git@github.com" -fi - -BRANCH="$(git config --get githubmerge.branch)" -if [[ "d$BRANCH" == "d" ]]; then - BRANCH="master" -fi - -TESTCMD="$(git config --get githubmerge.testcmd)" - -PULL="$1" - -if [[ "d$PULL" == "d" ]]; then - echo "Usage: $0 pullnumber [branch]" >&2 - exit 2 -fi - -if [[ "d$2" != "d" ]]; then - BRANCH="$2" -fi - -# Initialize source branches. -git checkout -q "$BRANCH" -if git fetch -q "$HOST":"$REPO" "+refs/pull/$PULL/*:refs/heads/pull/$PULL/*"; then - if ! git log -q -1 "refs/heads/pull/$PULL/head" >/dev/null 2>&1; then - echo "ERROR: Cannot find head of pull request #$PULL on $HOST:$REPO." >&2 - exit 3 - fi - if ! git log -q -1 "refs/heads/pull/$PULL/merge" >/dev/null 2>&1; then - echo "ERROR: Cannot find merge of pull request #$PULL on $HOST:$REPO." >&2 - exit 3 - fi -else - echo "ERROR: Cannot find pull request #$PULL on $HOST:$REPO." >&2 - exit 3 -fi -if git fetch -q "$HOST":"$REPO" +refs/heads/"$BRANCH":refs/heads/pull/"$PULL"/base; then - true -else - echo "ERROR: Cannot find branch $BRANCH on $HOST:$REPO." >&2 - exit 3 -fi -git checkout -q pull/"$PULL"/base -git branch -q -D pull/"$PULL"/local-merge 2>/dev/null -git checkout -q -b pull/"$PULL"/local-merge -TMPDIR="$(mktemp -d -t ghmXXXXX)" - -function cleanup() { - git checkout -q "$BRANCH" - git branch -q -D pull/"$PULL"/head 2>/dev/null - git branch -q -D pull/"$PULL"/base 2>/dev/null - git branch -q -D pull/"$PULL"/merge 2>/dev/null - git branch -q -D pull/"$PULL"/local-merge 2>/dev/null - rm -rf "$TMPDIR" -} - -# Create unsigned merge commit. -( - echo "Merge pull request #$PULL" - echo "" - git log --no-merges --topo-order --pretty='format:%h %s (%an)' pull/"$PULL"/base..pull/"$PULL"/head -)>"$TMPDIR/message" -if git merge -q --commit --no-edit --no-ff -m "$(<"$TMPDIR/message")" pull/"$PULL"/head; then - if [ "d$(git log --pretty='format:%s' -n 1)" != "dMerge pull request #$PULL" ]; then - echo "ERROR: Creating merge failed (already merged?)." >&2 - cleanup - exit 4 - fi -else - echo "ERROR: Cannot be merged cleanly." >&2 - git merge --abort - cleanup - exit 4 -fi - -# Run test command if configured. -if [[ "d$TESTCMD" != "d" ]]; then - # Go up to the repository's root. - while [ ! -d .git ]; do cd ..; done - if ! $TESTCMD; then - echo "ERROR: Running $TESTCMD failed." >&2 - cleanup - exit 5 - fi - # Show the created merge. - git diff pull/"$PULL"/merge..pull/"$PULL"/local-merge >"$TMPDIR"/diff - git diff pull/"$PULL"/base..pull/"$PULL"/local-merge - if [[ "$(<"$TMPDIR"/diff)" != "" ]]; then - echo "WARNING: merge differs from github!" >&2 - read -p "Type 'ignore' to continue. " -r >&2 - if [[ "d$REPLY" =~ ^d[iI][gG][nN][oO][rR][eE]$ ]]; then - echo "Difference with github ignored." >&2 - else - cleanup - exit 6 - fi - fi - read -p "Press 'd' to accept the diff. " -n 1 -r >&2 - echo - if [[ "d$REPLY" =~ ^d[dD]$ ]]; then - echo "Diff accepted." >&2 - else - echo "ERROR: Diff rejected." >&2 - cleanup - exit 6 - fi -else - # Verify the result. - echo "Dropping you on a shell so you can try building/testing the merged source." >&2 - echo "Run 'git diff HEAD~' to show the changes being merged." >&2 - echo "Type 'exit' when done." >&2 - if [[ -f /etc/debian_version ]]; then # Show pull number in prompt on Debian default prompt - export debian_chroot="$PULL" - fi - bash -i - read -p "Press 'm' to accept the merge. " -n 1 -r >&2 - echo - if [[ "d$REPLY" =~ ^d[Mm]$ ]]; then - echo "Merge accepted." >&2 - else - echo "ERROR: Merge rejected." >&2 - cleanup - exit 7 - fi -fi - -# Sign the merge commit. -read -p "Press 's' to sign off on the merge. " -n 1 -r >&2 -echo -if [[ "d$REPLY" =~ ^d[Ss]$ ]]; then - if [[ "$(git config --get user.signingkey)" == "" ]]; then - echo "ERROR: No GPG signing key set, not signing. Set one using:" >&2 - echo "git config --global user.signingkey <key>" >&2 - cleanup - exit 1 - else - if ! git commit -q --gpg-sign --amend --no-edit; then - echo "Error signing, exiting." - cleanup - exit 1 - fi - fi -else - echo "Not signing off on merge, exiting." - cleanup - exit 1 -fi - -# Clean up temporary branches, and put the result in $BRANCH. -git checkout -q "$BRANCH" -git reset -q --hard pull/"$PULL"/local-merge -cleanup - -# Push the result. -read -p "Type 'push' to push the result to $HOST:$REPO, branch $BRANCH. " -r >&2 -if [[ "d$REPLY" =~ ^d[Pp][Uu][Ss][Hh]$ ]]; then - git push "$HOST":"$REPO" refs/heads/"$BRANCH" -fi diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index e96eaa9c38..0319f739c4 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,7 +1,7 @@ #!/usr/bin/python2 ''' Perform basic ELF security checks on a series of executables. -Exit status will be 0 if succesful, and the program will be silent. +Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. Needs `readelf` (for ELF) and `objdump` (for PE). ''' @@ -94,7 +94,7 @@ def check_ELF_RELRO(executable): raise IOError('Error opening file') for line in stdout.split('\n'): tokens = line.split() - if len(tokens)>1 and tokens[1] == '(BIND_NOW)': + if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]): have_bindnow = True return have_gnu_relro and have_bindnow diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 93acfcdda4..4ad5136f79 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -42,9 +42,12 @@ MAX_VERSIONS = { 'GLIBCXX': (3,4,13), 'GLIBC': (2,11) } +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '_init', '__bss_start', '_fini' +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index f955e4a1f2..2b6e807b47 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -29,6 +29,8 @@ TX = 'tx' SOURCE_LANG = 'bitcoin_en.ts' # Directory with locale files LOCALE_DIR = 'src/qt/locale' +# Minimum number of messages for translation to be considered at all +MIN_NUM_MESSAGES = 10 def check_at_repository_root(): if not os.path.exists('.git'): @@ -37,7 +39,7 @@ def check_at_repository_root(): exit(1) def fetch_all_translations(): - if subprocess.call([TX, 'pull', '-f']): + if subprocess.call([TX, 'pull', '-f', '-a']): print('Error while fetching translations', file=sys.stderr) exit(1) @@ -70,7 +72,7 @@ def sanitize_string(s): '''Sanitize string for printing''' return s.replace('\n',' ') -def check_format_specifiers(source, translation, errors): +def check_format_specifiers(source, translation, errors, numerus): source_f = split_format_specifiers(find_format_specifiers(source)) # assert that no source messages contain both Qt and strprintf format specifiers # if this fails, go change the source as this is hacky and confusing! @@ -78,10 +80,13 @@ def check_format_specifiers(source, translation, errors): try: translation_f = split_format_specifiers(find_format_specifiers(translation)) except IndexError: - errors.append("Parse error in translation '%s'" % sanitize_string(translation)) + errors.append("Parse error in translation for '%s': '%s'" % (sanitize_string(source), sanitize_string(translation))) return False else: if source_f != translation_f: + if numerus and source_f == (set(), ['n']) and translation_f == (set(), []) and translation.find('%') == -1: + # Allow numerus translations to omit %n specifier (usually when it only has one possible value) + return True errors.append("Mismatch between '%s' and '%s'" % (sanitize_string(source), sanitize_string(translation))) return False return True @@ -148,7 +153,7 @@ def postprocess_translations(reduce_diff_hacks=False): if translation is None: continue errors = [] - valid = check_format_specifiers(source, translation, errors) + valid = check_format_specifiers(source, translation, errors, numerus) for error in errors: print('%s: %s' % (filename, error)) @@ -166,6 +171,15 @@ def postprocess_translations(reduce_diff_hacks=False): if translation_node.get('type') == 'unfinished': context.remove(message) + # check if document is (virtually) empty, and remove it if so + num_messages = 0 + for context in root.findall('context'): + for message in context.findall('message'): + num_messages += 1 + if num_messages < MIN_NUM_MESSAGES: + print('Removing %s, as it contains only %i messages' % (filepath, num_messages)) + continue + # write fixed-up tree # if diff reduction requested, replace some XML to 'sanitize' to qt formatting if reduce_diff_hacks: |