diff options
Diffstat (limited to 'test/lint')
| -rwxr-xr-x | test/lint/lint-filenames.sh | 24 | ||||
| -rwxr-xr-x | test/lint/lint-files.py | 207 | ||||
| -rwxr-xr-x | test/lint/lint-files.sh | 7 | ||||
| -rwxr-xr-x | test/lint/lint-shebang.sh | 24 |
4 files changed, 214 insertions, 48 deletions
diff --git a/test/lint/lint-filenames.sh b/test/lint/lint-filenames.sh deleted file mode 100755 index 3f7491cd2b..0000000000 --- a/test/lint/lint-filenames.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure only lowercase alphanumerics (a-z0-9), underscores (_), -# hyphens (-) and dots (.) are used in source code filenames. - -export LC_ALL=C - -EXIT_CODE=0 -OUTPUT=$(git ls-files --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]" | \ - grep -vE '^[a-z0-9_./-]+$' | \ - grep -vE '^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)') - -if [[ ${OUTPUT} != "" ]]; then - echo "Use only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.)" - echo "in source code filenames:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py new file mode 100755 index 0000000000..43ca882442 --- /dev/null +++ b/test/lint/lint-files.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This checks that all files in the repository have correct filenames and permissions +""" + +import os +import re +import sys +from subprocess import check_output +from typing import Optional, NoReturn + +CMD_ALL_FILES = "git ls-files --full-name" +CMD_SOURCE_FILES = 'git ls-files --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' +CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" +ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$" +ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$" +ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = ( + "^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)" +) +ALLOWED_PERMISSION_NON_EXECUTABLES = 644 +ALLOWED_PERMISSION_EXECUTABLES = 755 +ALLOWED_EXECUTABLE_SHEBANG = { + "py": [b"#!/usr/bin/env python3"], + "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"], +} + + +class FileMeta(object): + def __init__(self, file_path: str): + self.file_path = file_path + + @property + def extension(self) -> Optional[str]: + """ + Returns the file extension for a given filename string. + eg: + 'ci/lint_run_all.sh' -> 'sh' + 'ci/retry/retry' -> None + 'contrib/devtools/split-debug.sh.in' -> 'in' + """ + return str(os.path.splitext(self.file_path)[1].strip(".") or None) + + @property + def full_extension(self) -> Optional[str]: + """ + Returns the full file extension for a given filename string. + eg: + 'ci/lint_run_all.sh' -> 'sh' + 'ci/retry/retry' -> None + 'contrib/devtools/split-debug.sh.in' -> 'sh.in' + """ + filename_parts = self.file_path.split(os.extsep, 1) + try: + return filename_parts[1] + except IndexError: + return None + + @property + def permissions(self) -> int: + """ + Returns the octal file permission of the file + """ + return int(oct(os.stat(self.file_path).st_mode)[-3:]) + + +def check_all_filenames() -> int: + """ + Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase + alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames. + """ + # We avoid using rstrip() to ensure we catch filenames which accidentally include trailing whitespace + filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").split("\n") + filenames = [filename for filename in filenames if filename != ""] # removes the trailing empty list element + + filename_regex = re.compile(ALLOWED_FILENAME_REGEXP) + failed_tests = 0 + for filename in filenames: + if not filename_regex.match(filename): + print( + f"""File "{filename}" does not not match the allowed filename regexp ('{ALLOWED_FILENAME_REGEXP}').""" + ) + failed_tests += 1 + return failed_tests + + +def check_source_filenames() -> int: + """ + Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase + alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames. + + Additionally there is an exception regexp for directories or files which are excepted from matching this regexp. + """ + # We avoid using rstrip() to ensure we catch filenames which accidentally include trailing whitespace + filenames = check_output(CMD_SOURCE_FILES, shell=True).decode("utf8").split("\n") + filenames = [filename for filename in filenames if filename != ""] # removes the trailing empty list element + + filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP) + filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP) + failed_tests = 0 + for filename in filenames: + if not filename_regex.match(filename) and not filename_exception_regex.match(filename): + print( + f"""File "{filename}" does not not match the allowed source filename regexp ('{ALLOWED_SOURCE_FILENAME_REGEXP}'), or the exception regexp ({ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP}).""" + ) + failed_tests += 1 + return failed_tests + + +def check_all_file_permissions() -> int: + """ + Checks all files in the repository match an allowed executable or non-executable file permission octal. + + Additionally checks that for executable files, the file contains a shebang line + """ + filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").strip().split("\n") + failed_tests = 0 + for filename in filenames: + file_meta = FileMeta(filename) + if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES: + shebang = open(filename, "rb").readline().rstrip(b"\n") + + # For any file with executable permissions the first line must contain a shebang + if shebang[:2] != b"#!": + print( + f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" to make it non-executable.""" + ) + failed_tests += 1 + + # For certain file extensions that have been defined, we also check that the shebang conforms to a specific + # allowable set of shebangs + if file_meta.extension in ALLOWED_EXECUTABLE_SHEBANG.keys(): + if shebang not in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]: + print( + f"""File "{filename}" is missing expected shebang """ + + " or ".join( + [ + x.decode("utf-8") + for x in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension] + ] + ) + ) + failed_tests += 1 + + elif file_meta.permissions == ALLOWED_PERMISSION_NON_EXECUTABLES: + continue + else: + print( + f"""File "{filename}" has unexpected permission {file_meta.permissions}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (if executable).""" + ) + failed_tests += 1 + + return failed_tests + + +def check_shebang_file_permissions() -> int: + """ + Checks every file that contains a shebang line to ensure it has an executable permission + """ + filenames = check_output(CMD_SHEBANG_FILES, shell=True).decode("utf8").strip().split("\n") + + # The git grep command we use returns files which contain a shebang on any line within the file + # so we need to filter the list to only files with the shebang on the first line + filenames = [filename.split(":1:")[0] for filename in filenames if ":1:" in filename] + + failed_tests = 0 + for filename in filenames: + file_meta = FileMeta(filename) + if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES: + # These file types are typically expected to be sourced and not executed directly + if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]: + continue + + # *.py files which don't contain an `if __name__ == '__main__'` are not expected to be executed directly + if file_meta.extension == "py": + file_data = open(filename, "r", encoding="utf8").read() + if not re.search("""if __name__ == ['"]__main__['"]:""", file_data): + continue + + print( + f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (or remove the shebang line).""" + ) + failed_tests += 1 + return failed_tests + + +def main() -> NoReturn: + failed_tests = 0 + failed_tests += check_all_filenames() + failed_tests += check_source_filenames() + failed_tests += check_all_file_permissions() + failed_tests += check_shebang_file_permissions() + + if failed_tests: + print( + f"ERROR: There were {failed_tests} failed tests in the lint-files.py lint test. Please resolve the above errors." + ) + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh new file mode 100755 index 0000000000..1e115778bd --- /dev/null +++ b/test/lint/lint-files.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +export LC_ALL=C + +set -e +cd "$(dirname $0)/../.." +test/lint/lint-files.py diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh deleted file mode 100755 index 13ebdfd78a..0000000000 --- a/test/lint/lint-shebang.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Assert expected shebang lines - -export LC_ALL=C -EXIT_CODE=0 -for PYTHON_FILE in $(git ls-files -- "*.py"); do - if [[ $(head -c 2 "${PYTHON_FILE}") == "#!" && - $(head -n 1 "${PYTHON_FILE}") != "#!/usr/bin/env python3" ]]; then - echo "Missing shebang \"#!/usr/bin/env python3\" in ${PYTHON_FILE} (do not use python or python2)" - EXIT_CODE=1 - fi -done -for SHELL_FILE in $(git ls-files -- "*.sh"); do - if [[ $(head -n 1 "${SHELL_FILE}") != "#!/usr/bin/env bash" && - $(head -n 1 "${SHELL_FILE}") != "#!/bin/sh" ]]; then - echo "Missing expected shebang \"#!/usr/bin/env bash\" or \"#!/bin/sh\" in ${SHELL_FILE}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} |