diff options
Diffstat (limited to 'contrib/devtools')
| -rw-r--r-- | contrib/devtools/README.md | 72 | ||||
| -rwxr-xr-x | contrib/devtools/clang-format-diff.py | 4 | ||||
| -rwxr-xr-x | contrib/devtools/copyright_header.py | 34 | ||||
| -rwxr-xr-x | contrib/devtools/gen-manpages.sh | 2 | ||||
| -rwxr-xr-x | contrib/devtools/github-merge.py | 414 | ||||
| -rwxr-xr-x | contrib/devtools/symbol-check.py | 6 | ||||
| -rwxr-xr-x | contrib/devtools/update-translations.py | 215 |
7 files changed, 20 insertions, 727 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 4994d7f0a5..04fa02484f 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -89,66 +89,6 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh ``` -github-merge.py -=============== - -A small script to automate merging pull-requests securely and sign them with GPG. - -For example: - - ./github-merge.py 3077 - -(in any git repository) will help you merge pull request #3077 for the -bitcoin/bitcoin repository. - -What it does: -* Fetch master and the pull request. -* Locally construct a merge commit. -* Show the diff that merge results in. -* Ask you to verify the resulting source tree (so you can do a make -check or whatever). -* Ask you whether to GPG sign the merge commit. -* Ask you whether to push the result upstream. - -This means that there are no potential race conditions (where a -pullreq gets updated while you're reviewing it, but before you click -merge), and when using GPG signatures, that even a compromised GitHub -couldn't mess with the sources. - -Setup ---------- -Configuring the github-merge tool for the bitcoin repository is done in the following way: - - git config githubmerge.repository bitcoin/bitcoin - git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) - git config --global user.signingkey mykeyid - -Authentication (optional) --------------------------- - -The API request limit for unauthenticated requests is quite low, but the -limit for authenticated requests is much higher. If you start running -into rate limiting errors it can be useful to set an authentication token -so that the script can authenticate requests. - -- First, go to [Personal access tokens](https://github.com/settings/tokens). -- Click 'Generate new token'. -- Fill in an arbitrary token description. No further privileges are needed. -- Click the `Generate token` button at the bottom of the form. -- Copy the generated token (should be a hexadecimal string) - -Then do: - - git config --global user.ghtoken "pasted token" - -Create and verify timestamps of merge commits ---------------------------------------------- -To create or verify timestamps on the merge commits, install the OpenTimestamps -client via `pip3 install opentimestamps-client`. Then, download the gpg wrapper -`ots-git-gpg-wrapper.sh` and set it as git's `gpg.program`. See -[the ots git integration documentation](https://github.com/opentimestamps/opentimestamps-client/blob/master/doc/git-integration.md#usage) -for further details. - optimize-pngs.py ================ @@ -180,18 +120,6 @@ If there are 'unsupported' symbols, the return value will be 1 a list like this .../64/test_bitcoin: symbol std::out_of_range::~out_of_range() from unsupported version GLIBCXX_3.4.15 .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15 -update-translations.py -====================== - -Run this script from the root of the repository to update all translations from transifex. -It will do the following automatically: - -- fetch all translations -- post-process them into valid and committable format -- add missing translations to the build system (TODO) - -See doc/translation-process.md for more information. - circular-dependencies.py ======================== diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index f322b3a880..98eee67f43 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -106,7 +106,7 @@ def main(): filename = None lines_by_file = {} for line in sys.stdin: - match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) + match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) if match: filename = match.group(2) if filename is None: @@ -119,7 +119,7 @@ def main(): if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): continue - match = re.search('^@@.*\+(\d+)(,(\d+))?', line) + match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) if match: start_line = int(match.group(1)) line_count = 1 diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index f2987f2260..67e77bc63d 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -34,7 +34,7 @@ EXCLUDE_DIRS = [ "src/univalue/", ] -INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py'] INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) def applies_to_file(filename): @@ -71,7 +71,7 @@ def get_filenames_to_examine(base_directory): ################################################################################ -COPYRIGHT_WITH_C = 'Copyright \(c\)' +COPYRIGHT_WITH_C = r'Copyright \(c\)' COPYRIGHT_WITHOUT_C = 'Copyright' ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) @@ -85,23 +85,21 @@ ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE, ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) def compile_copyright_regex(copyright_style, year_style, name): - return re.compile('%s %s,? %s' % (copyright_style, year_style, name)) + return re.compile(r'%s %s,? %s( +\*)?\n' % (copyright_style, year_style, name)) EXPECTED_HOLDER_NAMES = [ - "Satoshi Nakamoto\n", - "The Bitcoin Core developers\n", - "Bitcoin Core Developers\n", - "BitPay Inc\.\n", - "University of Illinois at Urbana-Champaign\.\n", - "Pieter Wuille\n", - "Wladimir J. van der Laan\n", - "Jeff Garzik\n", - "Jan-Klaas Kollhof\n", - "Sam Rushing\n", - "ArtForz -- public domain half-a-node\n", - "Intel Corporation", - "The Zcash developers", - "Jeremy Rubin", + r"Satoshi Nakamoto", + r"The Bitcoin Core developers", + r"BitPay Inc\.", + r"University of Illinois at Urbana-Champaign\.", + r"Pieter Wuille", + r"Wladimir J\. van der Laan", + r"Jeff Garzik", + r"Jan-Klaas Kollhof", + r"ArtForz -- public domain half-a-node", + r"Intel Corporation ?", + r"The Zcash developers", + r"Jeremy Rubin", ] DOMINANT_STYLE_COMPILED = {} @@ -331,7 +329,7 @@ def write_file_lines(filename, file_lines): # update header years execution ################################################################################ -COPYRIGHT = 'Copyright \(c\)' +COPYRIGHT = r'Copyright \(c\)' YEAR = "20[0-9][0-9]" YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) HOLDER = 'The Bitcoin Core developers' diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 63b9847100..dbdb622877 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -16,7 +16,7 @@ BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} [ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 # The autodetected version git tag can screw up manpage output a little bit -BTCVER=($($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')) +read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" # Create a footer file with copyright content. # This gets autodetected fine for bitcoind if --version-string is not set, diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py deleted file mode 100755 index cd7a271e83..0000000000 --- a/contrib/devtools/github-merge.py +++ /dev/null @@ -1,414 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 Bitcoin Core Developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# This script will locally construct a merge commit for a pull request on a -# github repository, inspect it, sign it and optionally push it. - -# The following temporary branches are created/overwritten and deleted: -# * pull/$PULL/base (the current master we're merging onto) -# * pull/$PULL/head (the current state of the remote pull request) -# * pull/$PULL/merge (github's merge) -# * pull/$PULL/local-merge (our merge) - -# In case of a clean merge that is accepted by the user, the local branch with -# name $BRANCH is overwritten with the merged result, and optionally pushed. -import os -from sys import stdin,stdout,stderr -import argparse -import hashlib -import subprocess -import sys -import json -import codecs -from urllib.request import Request, urlopen -from urllib.error import HTTPError - -# External tools (can be overridden using environment) -GIT = os.getenv('GIT','git') -BASH = os.getenv('BASH','bash') - -# OS specific configuration for terminal attributes -ATTR_RESET = '' -ATTR_PR = '' -ATTR_NAME = '' -ATTR_WARN = '' -COMMIT_FORMAT = '%H %s (%an)%d' -if os.name == 'posix': # if posix, assume we can use basic terminal escapes - ATTR_RESET = '\033[0m' - ATTR_PR = '\033[1;36m' - ATTR_NAME = '\033[0;36m' - ATTR_WARN = '\033[1;31m' - COMMIT_FORMAT = '%C(bold blue)%H%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset' - -def git_config_get(option, default=None): - ''' - Get named configuration option from git repository. - ''' - try: - return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8') - except subprocess.CalledProcessError: - return default - -def get_response(req_url, ghtoken): - req = Request(req_url) - if ghtoken is not None: - req.add_header('Authorization', 'token ' + ghtoken) - return urlopen(req) - -def retrieve_json(req_url, ghtoken, use_pagination=False): - ''' - Retrieve json from github. - Return None if an error happens. - ''' - try: - reader = codecs.getreader('utf-8') - if not use_pagination: - return json.load(reader(get_response(req_url, ghtoken))) - - obj = [] - page_num = 1 - while True: - req_url_page = '{}?page={}'.format(req_url, page_num) - result = get_response(req_url_page, ghtoken) - obj.extend(json.load(reader(result))) - - link = result.headers.get('link', None) - if link is not None: - link_next = [l for l in link.split(',') if 'rel="next"' in l] - if len(link_next) > 0: - page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")]) - continue - break - return obj - except HTTPError as e: - error_message = e.read() - print('Warning: unable to retrieve pull information from github: %s' % e) - print('Detailed error: %s' % error_message) - return None - except Exception as e: - print('Warning: unable to retrieve pull information from github: %s' % e) - return None - -def retrieve_pr_info(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull - return retrieve_json(req_url,ghtoken) - -def retrieve_pr_comments(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments" - return retrieve_json(req_url,ghtoken,use_pagination=True) - -def retrieve_pr_reviews(repo,pull,ghtoken): - req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews" - return retrieve_json(req_url,ghtoken,use_pagination=True) - -def ask_prompt(text): - print(text,end=" ",file=stderr) - stderr.flush() - reply = stdin.readline().rstrip() - print("",file=stderr) - return reply - -def get_symlink_files(): - files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines()) - ret = [] - for f in files: - if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000: - ret.append(f.decode('utf-8').split("\t")[1]) - return ret - -def tree_sha512sum(commit='HEAD'): - # request metadata for entire tree, recursively - files = [] - blob_by_name = {} - for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): - name_sep = line.index(b'\t') - metadata = line[:name_sep].split() # perms, 'blob', blobid - assert(metadata[1] == b'blob') - name = line[name_sep+1:] - files.append(name) - blob_by_name[name] = metadata[2] - - files.sort() - # open connection to git-cat-file in batch mode to request data for all blobs - # this is much faster than launching it per file - p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) - overall = hashlib.sha512() - for f in files: - blob = blob_by_name[f] - # request blob - p.stdin.write(blob + b'\n') - p.stdin.flush() - # read header: blob, "blob", size - reply = p.stdout.readline().split() - assert(reply[0] == blob and reply[1] == b'blob') - size = int(reply[2]) - # hash the blob data - intern = hashlib.sha512() - ptr = 0 - while ptr < size: - bs = min(65536, size - ptr) - piece = p.stdout.read(bs) - if len(piece) == bs: - intern.update(piece) - else: - raise IOError('Premature EOF reading git cat-file output') - ptr += bs - dig = intern.hexdigest() - assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data - # update overall hash with file hash - overall.update(dig.encode("utf-8")) - overall.update(" ".encode("utf-8")) - overall.update(f) - overall.update("\n".encode("utf-8")) - p.stdin.close() - if p.wait(): - raise IOError('Non-zero return value executing git cat-file') - return overall.hexdigest() - -def get_acks_from_comments(head_commit, comments): - # Look for abbreviated commit id, because not everyone wants to type/paste - # the whole thing and the chance of collisions within a PR is small enough - head_abbrev = head_commit[0:6] - acks = [] - for c in comments: - review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_abbrev in l] - if review: - acks.append((c['user']['login'], review[0])) - return acks - -def make_acks_message(head_commit, acks): - if acks: - ack_str ='\n\nACKs for top commit:\n'.format(head_commit) - for name, msg in acks: - ack_str += ' {}:\n'.format(name) - ack_str += ' {}\n'.format(msg) - else: - ack_str ='\n\nTop commit has no ACKs.\n' - return ack_str - -def print_merge_details(pull, title, branch, base_branch, head_branch, acks): - print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) - if acks is not None: - if acks: - print('{}ACKs:{}'.format(ATTR_PR, ATTR_RESET)) - for (name, message) in acks: - print('* {} {}({}){}'.format(message, ATTR_NAME, name, ATTR_RESET)) - else: - print('{}Top commit has no ACKs!{}'.format(ATTR_WARN, ATTR_RESET)) - -def parse_arguments(): - epilog = ''' - In addition, you can set the following git configuration variables: - githubmerge.repository (mandatory), - user.signingkey (mandatory), - user.ghtoken (default: none). - githubmerge.host (default: git@github.com), - githubmerge.branch (no default), - githubmerge.testcmd (default: none). - ''' - parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', - epilog=epilog) - parser.add_argument('pull', metavar='PULL', type=int, nargs=1, - help='Pull request ID to merge') - parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', - default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')') - return parser.parse_args() - -def main(): - # Extract settings from git repo - repo = git_config_get('githubmerge.repository') - host = git_config_get('githubmerge.host','git@github.com') - opt_branch = git_config_get('githubmerge.branch',None) - testcmd = git_config_get('githubmerge.testcmd') - ghtoken = git_config_get('user.ghtoken') - signingkey = git_config_get('user.signingkey') - if repo is None: - print("ERROR: No repository configured. Use this command to set:", file=stderr) - print("git config githubmerge.repository <owner>/<repo>", file=stderr) - sys.exit(1) - if signingkey is None: - print("ERROR: No GPG signing key set. Set one using:",file=stderr) - print("git config --global user.signingkey <key>",file=stderr) - sys.exit(1) - - if host.startswith(('https:','http:')): - host_repo = host+"/"+repo+".git" - else: - host_repo = host+":"+repo - - # Extract settings from command line - args = parse_arguments() - pull = str(args.pull[0]) - - # Receive pull information from github - info = retrieve_pr_info(repo,pull,ghtoken) - if info is None: - sys.exit(1) - title = info['title'].strip() - body = info['body'].strip() - # precedence order for destination branch argument: - # - command line argument - # - githubmerge.branch setting - # - base branch for pull (as retrieved from github) - # - 'master' - branch = args.branch or opt_branch or info['base']['ref'] or 'master' - - # Initialize source branches - head_branch = 'pull/'+pull+'/head' - base_branch = 'pull/'+pull+'/base' - merge_branch = 'pull/'+pull+'/merge' - local_merge_branch = 'pull/'+pull+'/local-merge' - - devnull = open(os.devnull, 'w', encoding="utf8") - try: - subprocess.check_call([GIT,'checkout','-q',branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot check out branch %s." % (branch), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*', - '+refs/heads/'+branch+':refs/heads/'+base_branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot find pull request #%s or branch %s on %s." % (pull,branch,host_repo), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) - head_commit = subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8') - assert len(head_commit) == 40 - except subprocess.CalledProcessError: - print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr) - sys.exit(3) - try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) - except subprocess.CalledProcessError: - print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr) - sys.exit(3) - subprocess.check_call([GIT,'checkout','-q',base_branch]) - subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) - subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch]) - - try: - # Go up to the repository's root. - toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip() - os.chdir(toplevel) - # Create unsigned merge commit. - if title: - firstline = 'Merge #%s: %s' % (pull,title) - else: - firstline = 'Merge #%s' % (pull,) - message = firstline + '\n\n' - message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%H %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') - message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' - try: - subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','--no-gpg-sign','-m',message.encode('utf-8'),head_branch]) - except subprocess.CalledProcessError: - print("ERROR: Cannot be merged cleanly.",file=stderr) - subprocess.check_call([GIT,'merge','--abort']) - sys.exit(4) - logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8') - if logmsg.rstrip() != firstline.rstrip(): - print("ERROR: Creating merge failed (already merged?).",file=stderr) - sys.exit(4) - - symlink_files = get_symlink_files() - for f in symlink_files: - print("ERROR: File %s was a symlink" % f) - if len(symlink_files) > 0: - sys.exit(4) - - # Compute SHA512 of git tree (to be able to detect changes before sign-off) - try: - first_sha512 = tree_sha512sum() - except subprocess.CalledProcessError: - print("ERROR: Unable to compute tree hash") - sys.exit(4) - - print_merge_details(pull, title, branch, base_branch, head_branch, None) - print() - - # Run test command if configured. - if testcmd: - if subprocess.call(testcmd,shell=True): - print("ERROR: Running %s failed." % testcmd,file=stderr) - sys.exit(5) - - # Show the created merge. - diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch]) - subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch]) - if diff: - print("WARNING: merge differs from github!",file=stderr) - reply = ask_prompt("Type 'ignore' to continue.") - if reply.lower() == 'ignore': - print("Difference with github ignored.",file=stderr) - else: - sys.exit(6) - else: - # Verify the result manually. - print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) - print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr) - print("Type 'exit' when done.",file=stderr) - if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt - os.putenv('debian_chroot',pull) - subprocess.call([BASH,'-i']) - - second_sha512 = tree_sha512sum() - if first_sha512 != second_sha512: - print("ERROR: Tree hash changed unexpectedly",file=stderr) - sys.exit(8) - - # Retrieve PR comments and ACKs and add to commit message, store ACKs to print them with commit - # description - comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) - if comments is None: - print("ERROR: Could not fetch PR comments and reviews",file=stderr) - sys.exit(1) - acks = get_acks_from_comments(head_commit=head_commit, comments=comments) - message += make_acks_message(head_commit=head_commit, acks=acks) - # end message with SHA512 tree hash, then update message - message += '\n\nTree-SHA512: ' + first_sha512 - try: - subprocess.check_call([GIT,'commit','--amend','--no-gpg-sign','-m',message.encode('utf-8')]) - except subprocess.CalledProcessError: - print("ERROR: Cannot update message.", file=stderr) - sys.exit(4) - - # Sign the merge commit. - print_merge_details(pull, title, branch, base_branch, head_branch, acks) - while True: - reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() - if reply == 's': - try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) - break - except subprocess.CalledProcessError: - print("Error while signing, asking again.",file=stderr) - elif reply == 'x': - print("Not signing off on merge, exiting.",file=stderr) - sys.exit(1) - - # Put the result in branch. - subprocess.check_call([GIT,'checkout','-q',branch]) - subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch]) - finally: - # Clean up temporary branches. - subprocess.call([GIT,'checkout','-q',branch]) - subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull) - subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) - - # Push the result. - while True: - reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() - if reply == 'push': - subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) - break - elif reply == 'x': - sys.exit(1) - -if __name__ == '__main__': - main() - diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 7729dd7257..d8b684026c 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -67,8 +67,6 @@ ALLOWED_LIBRARIES = { 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker 'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker # bitcoin-qt only -'libX11-xcb.so.1', # part of X11 -'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing @@ -143,7 +141,7 @@ def read_libraries(filename): for line in stdout.splitlines(): tokens = line.split() if len(tokens)>2 and tokens[1] == '(NEEDED)': - match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + match = re.match(r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: @@ -173,5 +171,3 @@ if __name__ == '__main__': retval = 1 sys.exit(retval) - - diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py deleted file mode 100755 index 1b9d3a4c27..0000000000 --- a/contrib/devtools/update-translations.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Run this script from the root of the repository to update all translations from -transifex. -It will do the following automatically: - -- fetch all translations using the tx tool -- post-process them into valid and committable format - - remove invalid control characters - - remove location tags (makes diffs less noisy) - -TODO: -- auto-add new translations to the build system according to the translation process -''' -import subprocess -import re -import sys -import os -import io -import xml.etree.ElementTree as ET - -# Name of transifex tool -TX = 'tx' -# Name of source language file -SOURCE_LANG = 'bitcoin_en.ts' -# Directory with locale files -LOCALE_DIR = 'src/qt/locale' -# Minimum number of messages for translation to be considered at all -MIN_NUM_MESSAGES = 10 -# Regexp to check for Bitcoin addresses -ADDRESS_REGEXP = re.compile('([13]|bc1)[a-zA-Z0-9]{30,}') - -def check_at_repository_root(): - if not os.path.exists('.git'): - print('No .git directory found') - print('Execute this script at the root of the repository', file=sys.stderr) - sys.exit(1) - -def fetch_all_translations(): - if subprocess.call([TX, 'pull', '-f', '-a']): - print('Error while fetching translations', file=sys.stderr) - sys.exit(1) - -def find_format_specifiers(s): - '''Find all format specifiers in a string.''' - pos = 0 - specifiers = [] - while True: - percent = s.find('%', pos) - if percent < 0: - break - specifiers.append(s[percent+1]) - pos = percent+2 - return specifiers - -def split_format_specifiers(specifiers): - '''Split format specifiers between numeric (Qt) and others (strprintf)''' - numeric = [] - other = [] - for s in specifiers: - if s in {'1','2','3','4','5','6','7','8','9'}: - numeric.append(s) - else: - other.append(s) - - # If both numeric format specifiers and "others" are used, assume we're dealing - # with a Qt-formatted message. In the case of Qt formatting (see https://doc.qt.io/qt-5/qstring.html#arg) - # only numeric formats are replaced at all. This means "(percentage: %1%)" is valid, without needing - # any kind of escaping that would be necessary for strprintf. Without this, this function - # would wrongly detect '%)' as a printf format specifier. - if numeric: - other = [] - - # numeric (Qt) can be present in any order, others (strprintf) must be in specified order - return set(numeric),other - -def sanitize_string(s): - '''Sanitize string for printing''' - return s.replace('\n',' ') - -def check_format_specifiers(source, translation, errors, numerus): - source_f = split_format_specifiers(find_format_specifiers(source)) - # assert that no source messages contain both Qt and strprintf format specifiers - # if this fails, go change the source as this is hacky and confusing! - assert(not(source_f[0] and source_f[1])) - try: - translation_f = split_format_specifiers(find_format_specifiers(translation)) - except IndexError: - errors.append("Parse error in translation for '%s': '%s'" % (sanitize_string(source), sanitize_string(translation))) - return False - else: - if source_f != translation_f: - if numerus and source_f == (set(), ['n']) and translation_f == (set(), []) and translation.find('%') == -1: - # Allow numerus translations to omit %n specifier (usually when it only has one possible value) - return True - errors.append("Mismatch between '%s' and '%s'" % (sanitize_string(source), sanitize_string(translation))) - return False - return True - -def all_ts_files(suffix=''): - for filename in os.listdir(LOCALE_DIR): - # process only language files, and do not process source language - if not filename.endswith('.ts'+suffix) or filename == SOURCE_LANG+suffix: - continue - if suffix: # remove provided suffix - filename = filename[0:-len(suffix)] - filepath = os.path.join(LOCALE_DIR, filename) - yield(filename, filepath) - -FIX_RE = re.compile(b'[\x00-\x09\x0b\x0c\x0e-\x1f]') -def remove_invalid_characters(s): - '''Remove invalid characters from translation string''' - return FIX_RE.sub(b'', s) - -# Override cdata escape function to make our output match Qt's (optional, just for cleaner diffs for -# comparison, disable by default) -_orig_escape_cdata = None -def escape_cdata(text): - text = _orig_escape_cdata(text) - text = text.replace("'", ''') - text = text.replace('"', '"') - return text - -def contains_bitcoin_addr(text, errors): - if text is not None and ADDRESS_REGEXP.search(text) is not None: - errors.append('Translation "%s" contains a bitcoin address. This will be removed.' % (text)) - return True - return False - -def postprocess_translations(reduce_diff_hacks=False): - print('Checking and postprocessing...') - - if reduce_diff_hacks: - global _orig_escape_cdata - _orig_escape_cdata = ET._escape_cdata - ET._escape_cdata = escape_cdata - - for (filename,filepath) in all_ts_files(): - os.rename(filepath, filepath+'.orig') - - have_errors = False - for (filename,filepath) in all_ts_files('.orig'): - # pre-fixups to cope with transifex output - parser = ET.XMLParser(encoding='utf-8') # need to override encoding because 'utf8' is not understood only 'utf-8' - with open(filepath + '.orig', 'rb') as f: - data = f.read() - # remove control characters; this must be done over the entire file otherwise the XML parser will fail - data = remove_invalid_characters(data) - tree = ET.parse(io.BytesIO(data), parser=parser) - - # iterate over all messages in file - root = tree.getroot() - for context in root.findall('context'): - for message in context.findall('message'): - numerus = message.get('numerus') == 'yes' - source = message.find('source').text - translation_node = message.find('translation') - # pick all numerusforms - if numerus: - translations = [i.text for i in translation_node.findall('numerusform')] - else: - translations = [translation_node.text] - - for translation in translations: - if translation is None: - continue - errors = [] - valid = check_format_specifiers(source, translation, errors, numerus) and not contains_bitcoin_addr(translation, errors) - - for error in errors: - print('%s: %s' % (filename, error)) - - if not valid: # set type to unfinished and clear string if invalid - translation_node.clear() - translation_node.set('type', 'unfinished') - have_errors = True - - # Remove location tags - for location in message.findall('location'): - message.remove(location) - - # Remove entire message if it is an unfinished translation - if translation_node.get('type') == 'unfinished': - context.remove(message) - - # check if document is (virtually) empty, and remove it if so - num_messages = 0 - for context in root.findall('context'): - for message in context.findall('message'): - num_messages += 1 - if num_messages < MIN_NUM_MESSAGES: - print('Removing %s, as it contains only %i messages' % (filepath, num_messages)) - continue - - # write fixed-up tree - # if diff reduction requested, replace some XML to 'sanitize' to qt formatting - if reduce_diff_hacks: - out = io.BytesIO() - tree.write(out, encoding='utf-8') - out = out.getvalue() - out = out.replace(b' />', b'/>') - with open(filepath, 'wb') as f: - f.write(out) - else: - tree.write(filepath, encoding='utf-8') - return have_errors - -if __name__ == '__main__': - check_at_repository_root() - fetch_all_translations() - postprocess_translations() - |