Merge bitcoin/bitcoin#24895: lint: Convert lint-includes.sh to Python

67b41678c8284715c9cf96d6bdbde9a072b7a904 lint: Convert lint-includes.sh to Python (Dimitri)

Pull request description:

  A port of `test/lint/lint-includes.sh` to a Python-script as part of the request of #24783. Checked for output-consistency.

ACKs for top commit:
  KevinMusgrave:
    Tested ACK 67b41678c8284715c9cf96d6bdbde9a072b7a904

Tree-SHA512: 05b4b114dc101e571004aee8aea1480e4dda1dc645426100649e9cb81e56e8667f88d6d5646a9860ea1c7abc36754eda2a77ec10156c54b62db00e2c00b8ceae

2 files changed, 179 insertions, 103 deletions

diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py
new file mode 100755
index 0000000000..b29c7f8b4d
--- /dev/null
+++ b/test/lint/lint-includes.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check for duplicate includes.
+# Guard against accidental introduction of new Boost dependencies.
+# Check includes: Check for duplicate includes. Enforce bracket syntax includes.
+
+import os
+import re
+import sys
+
+from subprocess import check_output, CalledProcessError
+
+
+EXCLUDED_DIRS = ["src/leveldb/",
+                 "src/crc32c/",
+                 "src/secp256k1/",
+                 "src/minisketch/",
+                 "src/univalue/"]
+
+EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string.hpp",
+                           "boost/algorithm/string/classification.hpp",
+                           "boost/algorithm/string/replace.hpp",
+                           "boost/algorithm/string/split.hpp",
+                           "boost/date_time/posix_time/posix_time.hpp",
+                           "boost/multi_index/hashed_index.hpp",
+                           "boost/multi_index/ordered_index.hpp",
+                           "boost/multi_index/sequenced_index.hpp",
+                           "boost/multi_index_container.hpp",
+                           "boost/process.hpp",
+                           "boost/signals2/connection.hpp",
+                           "boost/signals2/optional_last_value.hpp",
+                           "boost/signals2/signal.hpp",
+                           "boost/test/included/unit_test.hpp",
+                           "boost/test/unit_test.hpp"]
+
+
+def get_toplevel():
+    return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n")
+
+
+def list_files_by_suffix(suffixes):
+    exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS]
+
+    files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines()
+
+    return [file for file in files_list if file.endswith(suffixes)]
+
+
+def find_duplicate_includes(include_list):
+    tempset = set()
+    duplicates = set()
+
+    for inclusion in include_list:
+        if inclusion in tempset:
+            duplicates.add(inclusion)
+        else:
+            tempset.add(inclusion)
+
+    return duplicates
+
+
+def find_included_cpps():
+    included_cpps = list()
+
+    try:
+        included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines()
+    except CalledProcessError as e:
+        if e.returncode > 1:
+            raise e
+
+    return included_cpps
+
+
+def find_extra_boosts():
+    included_boosts = list()
+    filtered_included_boost_set = set()
+    exclusion_set = set()
+
+    try:
+        included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines()
+    except CalledProcessError as e:
+        if e.returncode > 1:
+            raise e
+
+    for boost in included_boosts:
+        filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0])
+
+    for expected_boost in EXPECTED_BOOST_INCLUDES:
+        for boost in filtered_included_boost_set:
+            if expected_boost in boost:
+                exclusion_set.add(boost)
+
+    extra_boosts = set(filtered_included_boost_set.difference(exclusion_set))
+
+    return extra_boosts
+
+
+def find_quote_syntax_inclusions():
+    exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS]
+    quote_syntax_inclusions = list()
+
+    try:
+        quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines()
+    except CalledProcessError as e:
+        if e.returncode > 1:
+            raise e
+
+    return quote_syntax_inclusions
+
+
+def main():
+    exit_code = 0
+
+    os.chdir(get_toplevel())
+
+    # Check for duplicate includes
+    for filename in list_files_by_suffix((".cpp", ".h")):
+        with open(filename, "r", encoding="utf8") as file:
+            include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)]
+
+        duplicates = find_duplicate_includes(include_list)
+
+        if duplicates:
+            print(f"Duplicate include(s) in {filename}:")
+            for duplicate in duplicates:
+                print(duplicate)
+            print("")
+            exit_code = 1
+
+    # Check if code includes .cpp-files
+    included_cpps = find_included_cpps()
+
+    if included_cpps:
+        print("The following files #include .cpp files:")
+        for included_cpp in included_cpps:
+            print(included_cpp)
+        print("")
+        exit_code = 1
+
+    # Guard against accidental introduction of new Boost dependencies
+    extra_boosts = find_extra_boosts()
+
+    if extra_boosts:
+        for boost in extra_boosts:
+            print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:")
+            print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8"))
+        exit_code = 1
+
+    # Check if Boost dependencies are no longer used
+    for expected_boost in EXPECTED_BOOST_INCLUDES:
+        try:
+            check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")
+        except CalledProcessError as e:
+            if e.returncode > 1:
+                raise e
+            else:
+                print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. "
+                       "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py "
+                       "to make sure this dependency is not accidentally reintroduced.\n")
+                exit_code = 1
+
+    # Enforce bracket syntax includes
+    quote_syntax_inclusions = find_quote_syntax_inclusions()
+
+    if quote_syntax_inclusions:
+        print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:")
+        for quote_syntax_inclusion in quote_syntax_inclusions:
+            print(quote_syntax_inclusion)
+        exit_code = 1
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh
deleted file mode 100755
index 9e72831ee9..0000000000
--- a/test/lint/lint-includes.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2021 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Check for duplicate includes.
-# Guard against accidental introduction of new Boost dependencies.
-# Check includes: Check for duplicate includes. Enforce bracket syntax includes.
-
-export LC_ALL=C
-IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/"
-
-# cd to root folder of git repo for git ls-files to work properly
-cd "$(dirname "$0")/../.." || exit 1
-
-filter_suffix() {
-    git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}"
-}
-
-EXIT_CODE=0
-
-for HEADER_FILE in $(filter_suffix h); do
-    DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d)
-    if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then
-        echo "Duplicate include(s) in ${HEADER_FILE}:"
-        echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}"
-        echo
-        EXIT_CODE=1
-    fi
-done
-
-for CPP_FILE in $(filter_suffix cpp); do
-    DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d)
-    if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then
-        echo "Duplicate include(s) in ${CPP_FILE}:"
-        echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}"
-        echo
-        EXIT_CODE=1
-    fi
-done
-
-INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h")
-if [[ ${INCLUDED_CPP_FILES} != "" ]]; then
-    echo "The following files #include .cpp files:"
-    echo "${INCLUDED_CPP_FILES}"
-    echo
-    EXIT_CODE=1
-fi
-
-EXPECTED_BOOST_INCLUDES=(
-    boost/algorithm/string.hpp
-    boost/algorithm/string/classification.hpp
-    boost/algorithm/string/replace.hpp
-    boost/algorithm/string/split.hpp
-    boost/date_time/posix_time/posix_time.hpp
-    boost/multi_index/hashed_index.hpp
-    boost/multi_index/ordered_index.hpp
-    boost/multi_index/sequenced_index.hpp
-    boost/multi_index_container.hpp
-    boost/process.hpp
-    boost/signals2/connection.hpp
-    boost/signals2/optional_last_value.hpp
-    boost/signals2/signal.hpp
-    boost/test/included/unit_test.hpp
-    boost/test/unit_test.hpp
-)
-
-for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do
-    IS_EXPECTED_INCLUDE=0
-    for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do
-        if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then
-            IS_EXPECTED_INCLUDE=1
-            break
-        fi
-    done
-    if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then
-        EXIT_CODE=1
-        echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:"
-        git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h"
-        echo
-    fi
-done
-
-for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do
-    if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then
-        echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used."
-        echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0"
-        echo "to make sure this dependency is not accidentally reintroduced."
-        echo
-        EXIT_CODE=1
-    fi
-done
-
-QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}")
-if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then
-    echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:"
-    echo "${QUOTE_SYNTAX_INCLUDES}"
-    echo
-    EXIT_CODE=1
-fi
-
-exit ${EXIT_CODE}