diff options
Diffstat (limited to 'contrib/devtools')
| -rw-r--r-- | contrib/devtools/README.md | 76 | ||||
| -rwxr-xr-x | contrib/devtools/check-doc.py | 2 | ||||
| -rwxr-xr-x | contrib/devtools/clang-format-diff.py | 4 | ||||
| -rwxr-xr-x | contrib/devtools/clang-format.py | 62 | ||||
| -rwxr-xr-x | contrib/devtools/copyright_header.py | 610 | ||||
| -rwxr-xr-x | contrib/devtools/fix-copyright-headers.py | 46 | ||||
| -rwxr-xr-x | contrib/devtools/gen-manpages.sh | 29 | ||||
| -rwxr-xr-x | contrib/devtools/git-subtree-check.sh | 3 | ||||
| -rwxr-xr-x | contrib/devtools/github-merge.py | 4 | ||||
| -rwxr-xr-x | contrib/devtools/optimize-pngs.py | 5 | ||||
| -rwxr-xr-x | contrib/devtools/security-check.py | 53 | ||||
| -rwxr-xr-x | contrib/devtools/symbol-check.py | 2 | ||||
| -rwxr-xr-x | contrib/devtools/test-security-check.py | 6 | ||||
| -rwxr-xr-x | contrib/devtools/update-translations.py | 2 |
14 files changed, 762 insertions, 142 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index bb8b9246b8..34f2af6866 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -8,11 +8,6 @@ check-doc.py Check if all command line args are documented. The return value indicates the number of undocumented args. -clang-format.py -=============== - -A script to format cpp source code according to [.clang-format](../../src/.clang-format). This should only be applied to new files or files which are currently not actively developed on. Also, git subtrees are not subject to formatting. - clang-format-diff.py =================== @@ -25,20 +20,70 @@ the script should be called from the git root folder as follows. git diff -U0 HEAD~1.. | ./contrib/devtools/clang-format-diff.py -p1 -i -v ``` -fix-copyright-headers.py -======================== +copyright\_header.py +==================== -Every year newly updated files need to have its copyright headers updated to reflect the current year. -If you run this script from the root folder it will automatically update the year on the copyright header for all -source files if these have a git commit from the current year. +Provides utilities for managing copyright headers of `The Bitcoin Core +developers` in repository source files. It has three subcommands: -For example a file changed in 2015 (with 2015 being the current year): +``` +$ ./copyright_header.py report <base_directory> [verbose] +$ ./copyright_header.py update <base_directory> +$ ./copyright_header.py insert <file> +``` +Running these subcommands without arguments displays a usage string. -```// Copyright (c) 2009-2013 The Bitcoin Core developers``` +copyright\_header.py report \<base\_directory\> [verbose] +--------------------------------------------------------- -would be changed to: +Produces a report of all copyright header notices found inside the source files +of a repository. Useful to quickly visualize the state of the headers. +Specifying `verbose` will list the full filenames of files of each category. + +copyright\_header.py update \<base\_directory\> [verbose] +--------------------------------------------------------- +Updates all the copyright headers of `The Bitcoin Core developers` which were +changed in a year more recent than is listed. For example: +``` +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers +``` +where `<lastModifiedYear>` is obtained from the `git log` history. + +This subcommand also handles copyright headers that have only a single year. In +those cases: +``` +// Copyright (c) <year> The Bitcoin Core developers +``` +will be updated to: +``` +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers +``` +where the update is appropriate. + +copyright\_header.py insert \<file\> +------------------------------------ +Inserts a copyright header for `The Bitcoin Core developers` at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has `#!` starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be `<year_introduced>-<current_year>` where +`<year_introduced>` is according to the `git log` history. If +`<year_introduced>` is equal to `<current_year>`, it will be set as a single +year rather than two hyphenated years. + +If the file already has a copyright for `The Bitcoin Core developers`, the +script will exit. + +gen-manpages.sh +=============== -```// Copyright (c) 2009-2015 The Bitcoin Core developers``` +A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. +This requires help2man which can be found at: https://www.gnu.org/software/help2man/ git-subtree-check.sh ==================== @@ -51,8 +96,9 @@ maintained: * for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) * for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) * for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) +* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) -Usage: `git-subtree-check.sh DIR COMMIT` +Usage: `git-subtree-check.sh DIR (COMMIT)` `COMMIT` may be omitted, in which case `HEAD` is used. diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py index 9ea0131ac3..249214e931 100755 --- a/contrib/devtools/check-doc.py +++ b/contrib/devtools/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 The Bitcoin Core developers +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 13d2573b9f..7ea49b65e1 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -128,7 +128,7 @@ def main(): line_count = int(match.group(3)) if line_count == 0: continue - end_line = start_line + line_count - 1; + end_line = start_line + line_count - 1 lines_by_file.setdefault(filename, []).extend( ['-lines', str(start_line) + ':' + str(end_line)]) @@ -147,7 +147,7 @@ def main(): stderr=None, stdin=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: - sys.exit(p.returncode); + sys.exit(p.returncode) if not args.i: with open(filename) as f: diff --git a/contrib/devtools/clang-format.py b/contrib/devtools/clang-format.py deleted file mode 100755 index cee99047ac..0000000000 --- a/contrib/devtools/clang-format.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -''' -Wrapper script for clang-format - -Copyright (c) 2015 MarcoFalke -Copyright (c) 2015 The Bitcoin Core developers -Distributed under the MIT software license, see the accompanying -file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' - -import os -import sys -import subprocess - -tested_versions = ['3.6.0', '3.6.1', '3.6.2'] # A set of versions known to produce the same output -accepted_file_extensions = ('.h', '.cpp') # Files to format - -def check_clang_format_version(clang_format_exe): - try: - output = subprocess.check_output([clang_format_exe, '-version']) - for ver in tested_versions: - if ver in output: - print "Detected clang-format version " + ver - return - raise RuntimeError("Untested version: " + output) - except Exception as e: - print 'Could not verify version of ' + clang_format_exe + '.' - raise e - -def check_command_line_args(argv): - required_args = ['{clang-format-exe}', '{files}'] - example_args = ['clang-format-3.x', 'src/main.cpp', 'src/wallet/*'] - - if(len(argv) < len(required_args) + 1): - for word in (['Usage:', argv[0]] + required_args): - print word, - print '' - for word in (['E.g:', argv[0]] + example_args): - print word, - print '' - sys.exit(1) - -def run_clang_format(clang_format_exe, files): - for target in files: - if os.path.isdir(target): - for path, dirs, files in os.walk(target): - run_clang_format(clang_format_exe, (os.path.join(path, f) for f in files)) - elif target.endswith(accepted_file_extensions): - print "Format " + target - subprocess.check_call([clang_format_exe, '-i', '-style=file', target], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT) - else: - print "Skip " + target - -def main(argv): - check_command_line_args(argv) - clang_format_exe = argv[1] - files = argv[2:] - check_clang_format_version(clang_format_exe) - run_clang_format(clang_format_exe, files) - -if __name__ == "__main__": - main(sys.argv) diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py new file mode 100755 index 0000000000..9f35c378bf --- /dev/null +++ b/contrib/devtools/copyright_header.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import re +import fnmatch +import sys +import subprocess +import datetime +import os + +################################################################################ +# file filtering +################################################################################ + +EXCLUDE = [ + # libsecp256k1: + 'src/secp256k1/include/secp256k1.h', + 'src/secp256k1/include/secp256k1_ecdh.h', + 'src/secp256k1/include/secp256k1_recovery.h', + 'src/secp256k1/include/secp256k1_schnorr.h', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c', + 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c', + 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h', + # auto generated: + 'src/univalue/lib/univalue_escapes.h', + 'src/qt/bitcoinstrings.cpp', + 'src/chainparamsseeds.h', + # other external copyrights: + 'src/tinyformat.h', + 'src/leveldb/util/env_win.cc', + 'src/crypto/ctaes/bench.c', + 'qa/rpc-tests/test_framework/bignum.py', + # python init: + '*__init__.py', +] +EXCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in EXCLUDE])) + +INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] +INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE])) + +def applies_to_file(filename): + return ((EXCLUDE_COMPILED.match(filename) is None) and + (INCLUDE_COMPILED.match(filename) is not None)) + +################################################################################ +# obtain list of files in repo according to INCLUDE and EXCLUDE +################################################################################ + +GIT_LS_CMD = 'git ls-files' + +def call_git_ls(): + out = subprocess.check_output(GIT_LS_CMD.split(' ')) + return [f for f in out.decode("utf-8").split('\n') if f != ''] + +def get_filenames_to_examine(): + filenames = call_git_ls() + return sorted([filename for filename in filenames if + applies_to_file(filename)]) + +################################################################################ +# define and compile regexes for the patterns we are looking for +################################################################################ + + +COPYRIGHT_WITH_C = 'Copyright \(c\)' +COPYRIGHT_WITHOUT_C = 'Copyright' +ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) + +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +YEAR_LIST = '(%s)(, %s)+' % (YEAR, YEAR) +ANY_YEAR_STYLE = '(%s|%s)' % (YEAR_RANGE, YEAR_LIST) +ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE, + ANY_YEAR_STYLE)) + +ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) + +def compile_copyright_regex(copyright_style, year_style, name): + return re.compile('%s %s %s' % (copyright_style, year_style, name)) + +EXPECTED_HOLDER_NAMES = [ + "Satoshi Nakamoto\n", + "The Bitcoin Core developers\n", + "The Bitcoin Core developers \n", + "Bitcoin Core Developers\n", + "the Bitcoin Core developers\n", + "The Bitcoin developers\n", + "The LevelDB Authors\. All rights reserved\.\n", + "BitPay Inc\.\n", + "BitPay, Inc\.\n", + "University of Illinois at Urbana-Champaign\.\n", + "MarcoFalke\n", + "Pieter Wuille\n", + "Pieter Wuille +\*\n", + "Pieter Wuille, Gregory Maxwell +\*\n", + "Pieter Wuille, Andrew Poelstra +\*\n", + "Andrew Poelstra +\*\n", + "Wladimir J. van der Laan\n", + "Jeff Garzik\n", + "Diederik Huys, Pieter Wuille +\*\n", + "Thomas Daede, Cory Fields +\*\n", + "Jan-Klaas Kollhof\n", + "Sam Rushing\n", + "ArtForz -- public domain half-a-node\n", +] + +DOMINANT_STYLE_COMPILED = {} +YEAR_LIST_STYLE_COMPILED = {} +WITHOUT_C_STYLE_COMPILED = {} + +for holder_name in EXPECTED_HOLDER_NAMES: + DOMINANT_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) + YEAR_LIST_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) + WITHOUT_C_STYLE_COMPILED[holder_name] = ( + compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, + holder_name)) + +################################################################################ +# search file contents for copyright message of particular category +################################################################################ + +def get_count_of_copyrights_of_any_style_any_holder(contents): + return len(ANY_COPYRIGHT_COMPILED.findall(contents)) + +def file_has_dominant_style_copyright_for_holder(contents, holder_name): + match = DOMINANT_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_year_list_style_copyright_for_holder(contents, holder_name): + match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +def file_has_without_c_style_copyright_for_holder(contents, holder_name): + match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) + return match is not None + +################################################################################ +# get file info +################################################################################ + +def read_file(filename): + return open(os.path.abspath(filename), 'r').read() + +def gather_file_info(filename): + info = {} + info['filename'] = filename + c = read_file(filename) + info['contents'] = c + + info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c) + + info['classified_copyrights'] = 0 + info['dominant_style'] = {} + info['year_list_style'] = {} + info['without_c_style'] = {} + for holder_name in EXPECTED_HOLDER_NAMES: + has_dominant_style = ( + file_has_dominant_style_copyright_for_holder(c, holder_name)) + has_year_list_style = ( + file_has_year_list_style_copyright_for_holder(c, holder_name)) + has_without_c_style = ( + file_has_without_c_style_copyright_for_holder(c, holder_name)) + info['dominant_style'][holder_name] = has_dominant_style + info['year_list_style'][holder_name] = has_year_list_style + info['without_c_style'][holder_name] = has_without_c_style + if has_dominant_style or has_year_list_style or has_without_c_style: + info['classified_copyrights'] = info['classified_copyrights'] + 1 + return info + +################################################################################ +# report execution +################################################################################ + +SEPARATOR = '-'.join(['' for _ in range(80)]) + +def print_filenames(filenames, verbose): + if not verbose: + return + for filename in filenames: + print("\t%s" % filename) + +def print_report(file_infos, verbose): + print(SEPARATOR) + examined = [i['filename'] for i in file_infos] + print("%d files examined according to INCLUDE and EXCLUDE fnmatch rules" % + len(examined)) + print_filenames(examined, verbose) + + print(SEPARATOR) + print('') + zero_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 0] + print("%4d with zero copyrights" % len(zero_copyrights)) + print_filenames(zero_copyrights, verbose) + one_copyright = [i['filename'] for i in file_infos if + i['all_copyrights'] == 1] + print("%4d with one copyright" % len(one_copyright)) + print_filenames(one_copyright, verbose) + two_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 2] + print("%4d with two copyrights" % len(two_copyrights)) + print_filenames(two_copyrights, verbose) + three_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] == 3] + print("%4d with three copyrights" % len(three_copyrights)) + print_filenames(three_copyrights, verbose) + four_or_more_copyrights = [i['filename'] for i in file_infos if + i['all_copyrights'] >= 4] + print("%4d with four or more copyrights" % len(four_or_more_copyrights)) + print_filenames(four_or_more_copyrights, verbose) + print('') + print(SEPARATOR) + print('Copyrights with dominant style:\ne.g. "Copyright (c)" and ' + '"<year>" or "<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + dominant_style = [i['filename'] for i in file_infos if + i['dominant_style'][holder_name]] + if len(dominant_style) > 0: + print("%4d with '%s'" % (len(dominant_style), + holder_name.replace('\n', '\\n'))) + print_filenames(dominant_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with year list style:\ne.g. "Copyright (c)" and ' + '"<year1>, <year2>, ...":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + year_list_style = [i['filename'] for i in file_infos if + i['year_list_style'][holder_name]] + if len(year_list_style) > 0: + print("%4d with '%s'" % (len(year_list_style), + holder_name.replace('\n', '\\n'))) + print_filenames(year_list_style, verbose) + print('') + print(SEPARATOR) + print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "<year>" or ' + '"<startYear>-<endYear>":\n') + for holder_name in EXPECTED_HOLDER_NAMES: + without_c_style = [i['filename'] for i in file_infos if + i['without_c_style'][holder_name]] + if len(without_c_style) > 0: + print("%4d with '%s'" % (len(without_c_style), + holder_name.replace('\n', '\\n'))) + print_filenames(without_c_style, verbose) + + print('') + print(SEPARATOR) + + unclassified_copyrights = [i['filename'] for i in file_infos if + i['classified_copyrights'] < i['all_copyrights']] + print("%d with unexpected copyright holder names" % + len(unclassified_copyrights)) + print_filenames(unclassified_copyrights, verbose) + print(SEPARATOR) + +def exec_report(base_directory, verbose): + original_cwd = os.getcwd() + os.chdir(base_directory) + filenames = get_filenames_to_examine() + file_infos = [gather_file_info(f) for f in filenames] + print_report(file_infos, verbose) + os.chdir(original_cwd) + +################################################################################ +# report cmd +################################################################################ + +REPORT_USAGE = """ +Produces a report of all copyright header notices found inside the source files +of a repository. + +Usage: + $ ./copyright_header.py report <base_directory> [verbose] + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. + [verbose] - Includes a list of every file of each subcategory in the report. +""" + +def report_cmd(argv): + if len(argv) == 2: + sys.exit(REPORT_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad <base_directory>: %s" % base_directory) + + if len(argv) == 3: + verbose = False + elif argv[3] == 'verbose': + verbose = True + else: + sys.exit("*** unknown argument: %s" % argv[2]) + + exec_report(base_directory, verbose) + +################################################################################ +# query git for year of last change +################################################################################ + +GIT_LOG_CMD = "git log --pretty=format:%%ai %s" + +def call_git_log(filename): + out = subprocess.check_output((GIT_LOG_CMD % filename).split(' ')) + return out.decode("utf-8").split('\n') + +def get_git_change_years(filename): + git_log_lines = call_git_log(filename) + if len(git_log_lines) == 0: + return [datetime.date.today().year] + # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" + return [line.split(' ')[0].split('-')[0] for line in git_log_lines] + +def get_most_recent_git_change_year(filename): + return max(get_git_change_years(filename)) + +################################################################################ +# read and write to file +################################################################################ + +def read_file_lines(filename): + f = open(os.path.abspath(filename), 'r') + file_lines = f.readlines() + f.close() + return file_lines + +def write_file_lines(filename, file_lines): + f = open(os.path.abspath(filename), 'w') + f.write(''.join(file_lines)) + f.close() + +################################################################################ +# update header years execution +################################################################################ + +COPYRIGHT = 'Copyright \(c\)' +YEAR = "20[0-9][0-9]" +YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR) +HOLDER = 'The Bitcoin Core developers' +UPDATEABLE_LINE_COMPILED = re.compile(' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) + +def get_updatable_copyright_line(file_lines): + index = 0 + for line in file_lines: + if UPDATEABLE_LINE_COMPILED.search(line) is not None: + return index, line + index = index + 1 + return None, None + +def parse_year_range(year_range): + year_split = year_range.split('-') + start_year = year_split[0] + if len(year_split) == 1: + return start_year, start_year + return start_year, year_split[1] + +def year_range_to_str(start_year, end_year): + if start_year == end_year: + return start_year + return "%s-%s" % (start_year, end_year) + +def create_updated_copyright_line(line, last_git_change_year): + copyright_splitter = 'Copyright (c) ' + copyright_split = line.split(copyright_splitter) + # Preserve characters on line that are ahead of the start of the copyright + # notice - they are part of the comment block and vary from file-to-file. + before_copyright = copyright_split[0] + after_copyright = copyright_split[1] + + space_split = after_copyright.split(' ') + year_range = space_split[0] + start_year, end_year = parse_year_range(year_range) + if end_year == last_git_change_year: + return line + return (before_copyright + copyright_splitter + + year_range_to_str(start_year, last_git_change_year) + ' ' + + ' '.join(space_split[1:])) + +def update_updatable_copyright(filename): + file_lines = read_file_lines(filename) + index, line = get_updatable_copyright_line(file_lines) + if not line: + print_file_action_message(filename, "No updatable copyright.") + return + last_git_change_year = get_most_recent_git_change_year(filename) + new_line = create_updated_copyright_line(line, last_git_change_year) + if line == new_line: + print_file_action_message(filename, "Copyright up-to-date.") + return + file_lines[index] = new_line + write_file_lines(filename, file_lines) + print_file_action_message(filename, + "Copyright updated! -> %s" % last_git_change_year) + +def exec_update_header_year(base_directory): + original_cwd = os.getcwd() + os.chdir(base_directory) + for filename in get_filenames_to_examine(): + update_updatable_copyright(filename) + os.chdir(original_cwd) + +################################################################################ +# update cmd +################################################################################ + +UPDATE_USAGE = """ +Updates all the copyright headers of "The Bitcoin Core developers" which were +changed in a year more recent than is listed. For example: + +// Copyright (c) <firstYear>-<lastYear> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <firstYear>-<lastModifiedYear> The Bitcoin Core developers + +where <lastModifiedYear> is obtained from the 'git log' history. + +This subcommand also handles copyright headers that have only a single year. In those cases: + +// Copyright (c) <year> The Bitcoin Core developers + +will be updated to: + +// Copyright (c) <year>-<lastModifiedYear> The Bitcoin Core developers + +where the update is appropriate. + +Usage: + $ ./copyright_header.py update <base_directory> + +Arguments: + <base_directory> - The base directory of a bitcoin source code repository. +""" + +def print_file_action_message(filename, action): + print("%-52s %s" % (filename, action)) + +def update_cmd(argv): + if len(argv) != 3: + sys.exit(UPDATE_USAGE) + + base_directory = argv[2] + if not os.path.exists(base_directory): + sys.exit("*** bad base_directory: %s" % base_directory) + exec_update_header_year(base_directory) + +################################################################################ +# inserted copyright header format +################################################################################ + +def get_header_lines(header, start_year, end_year): + lines = header.split('\n')[1:-1] + lines[0] = lines[0] % year_range_to_str(start_year, end_year) + return [line + '\n' for line in lines] + +CPP_HEADER = ''' +// Copyright (c) %s The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_cpp_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) + +PYTHON_HEADER = ''' +# Copyright (c) %s The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' + +def get_python_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) + +################################################################################ +# query git for year of last change +################################################################################ + +def get_git_change_year_range(filename): + years = get_git_change_years(filename) + return min(years), max(years) + +################################################################################ +# check for existing core copyright +################################################################################ + +def file_already_has_core_copyright(file_lines): + index, _ = get_updatable_copyright_line(file_lines) + return index != None + +################################################################################ +# insert header execution +################################################################################ + +def file_has_hashbang(file_lines): + if len(file_lines) < 1: + return False + if len(file_lines[0]) <= 2: + return False + return file_lines[0][:2] == '#!' + +def insert_python_header(filename, file_lines, start_year, end_year): + if file_has_hashbang(file_lines): + insert_idx = 1 + else: + insert_idx = 0 + header_lines = get_python_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(insert_idx, line) + write_file_lines(filename, file_lines) + +def insert_cpp_header(filename, file_lines, start_year, end_year): + header_lines = get_cpp_header_lines_to_insert(start_year, end_year) + for line in header_lines: + file_lines.insert(0, line) + write_file_lines(filename, file_lines) + +def exec_insert_header(filename, style): + file_lines = read_file_lines(filename) + if file_already_has_core_copyright(file_lines): + sys.exit('*** %s already has a copyright by The Bitcoin Core developers' + % (filename)) + start_year, end_year = get_git_change_year_range(filename) + if style == 'python': + insert_python_header(filename, file_lines, start_year, end_year) + else: + insert_cpp_header(filename, file_lines, start_year, end_year) + +################################################################################ +# insert cmd +################################################################################ + +INSERT_USAGE = """ +Inserts a copyright header for "The Bitcoin Core developers" at the top of the +file in either Python or C++ style as determined by the file extension. If the +file is a Python file and it has a '#!' starting the first line, the header is +inserted in the line below it. + +The copyright dates will be set to be: + +"<year_introduced>-<current_year>" + +where <year_introduced> is according to the 'git log' history. If +<year_introduced> is equal to <current_year>, the date will be set to be: + +"<current_year>" + +If the file already has a copyright for "The Bitcoin Core developers", the +script will exit. + +Usage: + $ ./copyright_header.py insert <file> + +Arguments: + <file> - A source file in the bitcoin repository. +""" + +def insert_cmd(argv): + if len(argv) != 3: + sys.exit(INSERT_USAGE) + + filename = argv[2] + if not os.path.isfile(filename): + sys.exit("*** bad filename: %s" % filename) + _, extension = os.path.splitext(filename) + if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: + sys.exit("*** cannot insert for file extension %s" % extension) + + if extension == '.py': + style = 'python' + else: + style = 'cpp' + exec_insert_header(filename, style) + +################################################################################ +# UI +################################################################################ + +USAGE = """ +copyright_header.py - utilities for managing copyright headers of 'The Bitcoin +Core developers' in repository source files. + +Usage: + $ ./copyright_header <subcommand> + +Subcommands: + report + update + insert + +To see subcommand usage, run them without arguments. +""" + +SUBCOMMANDS = ['report', 'update', 'insert'] + +if __name__ == "__main__": + if len(sys.argv) == 1: + sys.exit(USAGE) + subcommand = sys.argv[1] + if subcommand not in SUBCOMMANDS: + sys.exit(USAGE) + if subcommand == 'report': + report_cmd(sys.argv) + elif subcommand == 'update': + update_cmd(sys.argv) + elif subcommand == 'insert': + insert_cmd(sys.argv) diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py deleted file mode 100755 index b6414a551f..0000000000 --- a/contrib/devtools/fix-copyright-headers.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -''' -Run this script to update all the copyright headers of files -that were changed this year. - -For example: - -// Copyright (c) 2009-2012 The Bitcoin Core developers - -it will change it to - -// Copyright (c) 2009-2015 The Bitcoin Core developers -''' -import os -import time -import re - -year = time.gmtime()[0] -CMD_GIT_DATE = 'git log --format=@%%at -1 %s | date +"%%Y" -u -f -' -CMD_REGEX= "perl -pi -e 's/(20\d\d)(?:-20\d\d)? The Bitcoin/$1-%s The Bitcoin/' %s" -REGEX_CURRENT= re.compile("%s The Bitcoin" % year) -CMD_LIST_FILES= "find %s | grep %s" - -FOLDERS = ["./qa", "./src"] -EXTENSIONS = [".cpp",".h", ".py"] - -def get_git_date(file_path): - r = os.popen(CMD_GIT_DATE % file_path) - for l in r: - # Result is one line, so just return - return l.replace("\n","") - return "" - -n=1 -for folder in FOLDERS: - for extension in EXTENSIONS: - for file_path in os.popen(CMD_LIST_FILES % (folder, extension)): - file_path = os.getcwd() + file_path[1:-1] - if file_path.endswith(extension): - git_date = get_git_date(file_path) - if str(year) == git_date: - # Only update if current year is not found - if REGEX_CURRENT.search(open(file_path, "r").read()) is None: - print n,"Last git edit", git_date, "-", file_path - os.popen(CMD_REGEX % (year,file_path)) - n = n + 1 diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh new file mode 100755 index 0000000000..967717e1e0 --- /dev/null +++ b/contrib/devtools/gen-manpages.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} +SRCDIR=${SRCDIR:-$TOPDIR/src} +MANDIR=${MANDIR:-$TOPDIR/doc/man} + +BITCOIND=${BITCOIND:-$SRCDIR/bitcoind} +BITCOINCLI=${BITCOINCLI:-$SRCDIR/bitcoin-cli} +BITCOINTX=${BITCOINTX:-$SRCDIR/bitcoin-tx} +BITCOINQT=${BITCOINQT:-$SRCDIR/qt/bitcoin-qt} + +[ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 + +# The autodetected version git tag can screw up manpage output a little bit +BTCVER=($($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')) + +# Create a footer file with copyright content. +# This gets autodetected fine for bitcoind if --version-string is not set, +# but has different outcomes for bitcoin-qt and bitcoin-cli. +echo "[COPYRIGHT]" > footer.h2m +$BITCOIND --version | sed -n '1!p' >> footer.h2m + +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $BITCOINQT; do + cmdname="${cmd##*/}" + help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} + sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 +done + +rm -f footer.h2m diff --git a/contrib/devtools/git-subtree-check.sh b/contrib/devtools/git-subtree-check.sh index 1cb82fe682..2384d66cad 100755 --- a/contrib/devtools/git-subtree-check.sh +++ b/contrib/devtools/git-subtree-check.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. DIR="$1" COMMIT="$2" diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index f82362fe41..0cee0921b1 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 Bitcoin Core Developers +# Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ # In case of a clean merge that is accepted by the user, the local branch with # name $BRANCH is overwritten with the merged result, and optionally pushed. from __future__ import division,print_function,unicode_literals -import os,sys +import os from sys import stdin,stdout,stderr import argparse import subprocess diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py index 799e0cc7d0..9286ab731f 100755 --- a/contrib/devtools/optimize-pngs.py +++ b/contrib/devtools/optimize-pngs.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text). #pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text @@ -36,7 +39,7 @@ for folder in folders: if extension.lower() == '.png': print("optimizing "+file+"..."), file_path = os.path.join(absFolder, file) - fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)}; + fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)} fileMetaMap['contentHashPre'] = content_hash(file_path) pngCrushOutput = "" diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 301fea85c1..c90541e271 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,4 +1,7 @@ -#!/usr/bin/python2 +#!/usr/bin/env python +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Perform basic ELF security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. @@ -12,6 +15,7 @@ import os READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') +NONFATAL = {'HIGH_ENTROPY_VA'} # checks which are non-fatal for now but only generate a warning def check_ELF_PIE(executable): ''' @@ -114,26 +118,50 @@ def check_ELF_Canary(executable): def get_PE_dll_characteristics(executable): ''' - Get PE DllCharacteristics bits + Get PE DllCharacteristics bits. + Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' + and bits is the DllCharacteristics value. ''' p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') + arch = '' + bits = 0 for line in stdout.split('\n'): tokens = line.split() + if len(tokens)>=2 and tokens[0] == 'architecture:': + arch = tokens[1].rstrip(',') if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': - return int(tokens[1],16) - return 0 + bits = int(tokens[1],16) + return (arch,bits) +IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 +IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 +IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 -def check_PE_PIE(executable): +def check_PE_DYNAMIC_BASE(executable): '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - return bool(get_PE_dll_characteristics(executable) & 0x40) + (arch,bits) = get_PE_dll_characteristics(executable) + reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE + return (bits & reqbits) == reqbits + +# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE +# to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(executable): + '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + (arch,bits) = get_PE_dll_characteristics(executable) + if arch == 'i386:x86-64': + reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA + else: # Unnecessary on 32-bit + assert(arch == 'i386') + reqbits = 0 + return (bits & reqbits) == reqbits def check_PE_NX(executable): '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - return bool(get_PE_dll_characteristics(executable) & 0x100) + (arch,bits) = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT CHECKS = { 'ELF': [ @@ -143,7 +171,8 @@ CHECKS = { ('Canary', check_ELF_Canary) ], 'PE': [ - ('PIE', check_PE_PIE), + ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), + ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_PE_NX) ] } @@ -168,12 +197,18 @@ if __name__ == '__main__': continue failed = [] + warning = [] for (name, func) in CHECKS[etype]: if not func(filename): - failed.append(name) + if name in NONFATAL: + warning.append(name) + else: + failed.append(name) if failed: print('%s: failed %s' % (filename, ' '.join(failed))) retval = 1 + if warning: + print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: print('%s: cannot open' % filename) retval = 1 diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index e26c0fbb94..8f8685006e 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index fed7626aab..18f9835faa 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,10 +1,12 @@ -#!/usr/bin/python2 +#!/usr/bin/env python2 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' from __future__ import division,print_function import subprocess -import sys import unittest def write_testcode(filename): diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index 2b6e807b47..78b9f9d179 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. |