diff options
Diffstat (limited to 'contrib/devtools')
| -rw-r--r-- | contrib/devtools/README.md | 6 | ||||
| -rwxr-xr-x | contrib/devtools/github-merge.py | 52 | ||||
| -rwxr-xr-x | contrib/devtools/security-check.py | 34 | ||||
| -rwxr-xr-x | contrib/devtools/symbol-check.py | 63 |
4 files changed, 86 insertions, 69 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 1103ca86c5..bb8b9246b8 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -48,9 +48,9 @@ the commit it claims to have been updated to. To use, make sure that you have fetched the upstream repository branch in which the subtree is maintained: -* for `src/secp256k1`: https://github.com/bitcoin/secp256k1.git (branch master) -* for `src/leveldb`: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork) -* for `src/univalue`: https://github.com/bitcoin/univalue.git (branch master) +* for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) +* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) +* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) Usage: `git-subtree-check.sh DIR COMMIT` diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index c8dcaae268..f82362fe41 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2016 Bitcoin Core Developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,11 @@ import os,sys from sys import stdin,stdout,stderr import argparse import subprocess +import json,codecs +try: + from urllib.request import Request,urlopen +except: + from urllib2 import Request,urlopen # External tools (can be overridden using environment) GIT = os.getenv('GIT','git') @@ -38,38 +43,39 @@ def git_config_get(option, default=None): Get named configuration option from git repository. ''' try: - return subprocess.check_output([GIT,'config','--get',option]).rstrip() + return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8') except subprocess.CalledProcessError as e: return default -def retrieve_pr_title(repo,pull): +def retrieve_pr_info(repo,pull): ''' - Retrieve pull request title from github. + Retrieve pull request information from github. Return None if no title can be found, or an error happens. ''' - import urllib2,json try: - req = urllib2.Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) - result = urllib2.urlopen(req) - result = json.load(result) - return result['title'] + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + result = urlopen(req) + reader = codecs.getreader('utf-8') + obj = json.load(reader(result)) + return obj except Exception as e: - print('Warning: unable to retrieve pull title from github: %s' % e) + print('Warning: unable to retrieve pull information from github: %s' % e) return None def ask_prompt(text): print(text,end=" ",file=stderr) + stderr.flush() reply = stdin.readline().rstrip() print("",file=stderr) return reply -def parse_arguments(branch): +def parse_arguments(): epilog = ''' In addition, you can set the following git configuration variables: githubmerge.repository (mandatory), user.signingkey (mandatory), githubmerge.host (default: git@github.com), - githubmerge.branch (default: master), + githubmerge.branch (no default), githubmerge.testcmd (default: none). ''' parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', @@ -77,14 +83,14 @@ def parse_arguments(branch): parser.add_argument('pull', metavar='PULL', type=int, nargs=1, help='Pull request ID to merge') parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', - default=branch, help='Branch to merge against (default: '+branch+')') + default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')') return parser.parse_args() def main(): # Extract settings from git repo repo = git_config_get('githubmerge.repository') host = git_config_get('githubmerge.host','git@github.com') - branch = git_config_get('githubmerge.branch','master') + opt_branch = git_config_get('githubmerge.branch',None) testcmd = git_config_get('githubmerge.testcmd') signingkey = git_config_get('user.signingkey') if repo is None: @@ -99,9 +105,20 @@ def main(): host_repo = host+":"+repo # shortcut for push/pull target # Extract settings from command line - args = parse_arguments(branch) + args = parse_arguments() pull = str(args.pull[0]) - branch = args.branch + + # Receive pull information from github + info = retrieve_pr_info(repo,pull) + if info is None: + exit(1) + title = info['title'] + # precedence order for destination branch argument: + # - command line argument + # - githubmerge.branch setting + # - base branch for pull (as retrieved from github) + # - 'master' + branch = args.branch or opt_branch or info['base']['ref'] or 'master' # Initialize source branches head_branch = 'pull/'+pull+'/head' @@ -141,7 +158,6 @@ def main(): try: # Create unsigned merge commit. - title = retrieve_pr_title(repo,pull) if title: firstline = 'Merge #%s: %s' % (pull,title) else: @@ -159,7 +175,7 @@ def main(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) - print('%s#%s%s %s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title)) + print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) print() # Run test command if configured. diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 0319f739c4..301fea85c1 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -5,7 +5,7 @@ Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. Needs `readelf` (for ELF) and `objdump` (for PE). ''' -from __future__ import division,print_function +from __future__ import division,print_function,unicode_literals import subprocess import sys import os @@ -23,9 +23,9 @@ def check_ELF_PIE(executable): raise IOError('Error opening file') ok = False - for line in stdout.split('\n'): + for line in stdout.split(b'\n'): line = line.split() - if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': + if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN': ok = True return ok @@ -38,17 +38,17 @@ def get_ELF_program_headers(executable): in_headers = False count = 0 headers = [] - for line in stdout.split('\n'): - if line.startswith('Program Headers:'): + for line in stdout.split(b'\n'): + if line.startswith(b'Program Headers:'): in_headers = True - if line == '': + if line == b'': in_headers = False if in_headers: if count == 1: # header line - ofs_typ = line.find('Type') - ofs_offset = line.find('Offset') - ofs_flags = line.find('Flg') - ofs_align = line.find('Align') + ofs_typ = line.find(b'Type') + ofs_offset = line.find(b'Offset') + ofs_flags = line.find(b'Flg') + ofs_align = line.find(b'Align') if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: raise ValueError('Cannot parse elfread -lW output') elif count > 1: @@ -65,9 +65,9 @@ def check_ELF_NX(executable): have_wx = False have_gnu_stack = False for (typ, flags) in get_ELF_program_headers(executable): - if typ == 'GNU_STACK': + if typ == b'GNU_STACK': have_gnu_stack = True - if 'W' in flags and 'E' in flags: # section is both writable and executable + if b'W' in flags and b'E' in flags: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx @@ -84,7 +84,7 @@ def check_ELF_RELRO(executable): # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 - if typ == 'GNU_RELRO': + if typ == b'GNU_RELRO': have_gnu_relro = True have_bindnow = False @@ -92,9 +92,9 @@ def check_ELF_RELRO(executable): (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') - for line in stdout.split('\n'): + for line in stdout.split(b'\n'): tokens = line.split() - if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]): + if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]): have_bindnow = True return have_gnu_relro and have_bindnow @@ -107,8 +107,8 @@ def check_ELF_Canary(executable): if p.returncode: raise IOError('Error opening file') ok = False - for line in stdout.split('\n'): - if '__stack_chk_fail' in line: + for line in stdout.split(b'\n'): + if b'__stack_chk_fail' in line: ok = True return ok diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 4ad5136f79..e26c0fbb94 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -11,7 +11,7 @@ Example usage: find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py ''' -from __future__ import division, print_function +from __future__ import division, print_function, unicode_literals import subprocess import re import sys @@ -47,28 +47,28 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used' +b'_edata', b'_end', b'_init', b'__bss_start', b'_fini', b'_IO_stdin_used' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -'libgcc_s.so.1', # GCC base support -'libc.so.6', # C library -'libpthread.so.0', # threading -'libanl.so.1', # DNS resolve -'libm.so.6', # math library -'librt.so.1', # real-time (clock) -'ld-linux-x86-64.so.2', # 64-bit dynamic linker -'ld-linux.so.2', # 32-bit dynamic linker +b'libgcc_s.so.1', # GCC base support +b'libc.so.6', # C library +b'libpthread.so.0', # threading +b'libanl.so.1', # DNS resolve +b'libm.so.6', # math library +b'librt.so.1', # real-time (clock) +b'ld-linux-x86-64.so.2', # 64-bit dynamic linker +b'ld-linux.so.2', # 32-bit dynamic linker # bitcoin-qt only -'libX11-xcb.so.1', # part of X11 -'libX11.so.6', # part of X11 -'libxcb.so.1', # part of X11 -'libfontconfig.so.1', # font support -'libfreetype.so.6', # font parsing -'libdl.so.2' # programming interface to dynamic linker +b'libX11-xcb.so.1', # part of X11 +b'libX11.so.6', # part of X11 +b'libxcb.so.1', # part of X11 +b'libfontconfig.so.1', # font support +b'libfreetype.so.6', # font parsing +b'libdl.so.2' # programming interface to dynamic linker } class CPPFilt(object): @@ -81,7 +81,8 @@ class CPPFilt(object): self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE) def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') + self.proc.stdin.write(mangled + b'\n') + self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() def close(self): @@ -99,24 +100,24 @@ def read_symbols(executable, imports=True): if p.returncode: raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) syms = [] - for line in stdout.split('\n'): + for line in stdout.split(b'\n'): line = line.split() - if len(line)>7 and re.match('[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition('@') - is_import = line[6] == 'UND' - if version.startswith('@'): + if len(line)>7 and re.match(b'[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition(b'@') + is_import = line[6] == b'UND' + if version.startswith(b'@'): version = version[1:] if is_import == imports: syms.append((sym, version)) return syms def check_version(max_versions, version): - if '_' in version: - (lib, _, ver) = version.rpartition('_') + if b'_' in version: + (lib, _, ver) = version.rpartition(b'_') else: lib = version ver = '0' - ver = tuple([int(x) for x in ver.split('.')]) + ver = tuple([int(x) for x in ver.split(b'.')]) if not lib in max_versions: return False return ver <= max_versions[lib] @@ -127,10 +128,10 @@ def read_libraries(filename): if p.returncode: raise IOError('Error opening file') libraries = [] - for line in stdout.split('\n'): + for line in stdout.split(b'\n'): tokens = line.split() - if len(tokens)>2 and tokens[1] == '(NEEDED)': - match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if len(tokens)>2 and tokens[1] == b'(NEEDED)': + match = re.match(b'^Shared library: \[(.*)\]$', b' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: @@ -144,18 +145,18 @@ if __name__ == '__main__': # Check imported symbols for sym,version in read_symbols(filename, True): if version and not check_version(MAX_VERSIONS, version): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym).decode('utf-8'), version.decode('utf-8'))) retval = 1 # Check exported symbols for sym,version in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym).decode('utf-8'))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + print('%s: NEEDED library %s is not allowed' % (filename, library_name.decode('utf-8'))) retval = 1 exit(retval) |