Merge #13448: Add linter: Make sure we explicitly open all text files using UTF-8 encoding in Python

c8176b3cc7556d7bcec39a55ae4d6ba16453baaa Add linter: Make sure we explicitly open all text files using UTF-8 or ASCII encoding in Python (practicalswift)
634bd970013eca90f4b4c1f9044eec8c97ba62c2 Explicitly specify encoding when opening text files in Python code (practicalswift)

Pull request description:

  Add linter: Make sure we explicitly open all text files using UTF-8 encoding in Python.

  As requested by @laanwj in #13440.

Tree-SHA512: 1651c00fe220ceb273324abd6703aee504029b96c7ef0e3029145901762c733c9b9d24927da281394fd4681a5bff774336c04eed01fafea997bb32192c334c06

20 files changed, 57 insertions, 38 deletions

diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py
index d544d5c371..abfa5ed5ae 100755
--- a/contrib/devtools/circular-dependencies.py
+++ b/contrib/devtools/circular-dependencies.py
@@ -37,7 +37,7 @@ for arg in sys.argv[1:]:
 # TODO: implement support for multiple include directories
 for arg in sorted(files.keys()):
     module = files[arg]
-    with open(arg, 'r') as f:
+    with open(arg, 'r', encoding="utf8") as f:
         for line in f:
             match = RE.match(line)
             if match:
diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py
index 5402870fba..77e845a9b4 100755
--- a/contrib/devtools/clang-format-diff.py
+++ b/contrib/devtools/clang-format-diff.py
@@ -152,7 +152,7 @@ def main():
       sys.exit(p.returncode)
 
     if not args.i:
-      with open(filename) as f:
+      with open(filename, encoding="utf8") as f:
         code = f.readlines()
       formatted_code = io.StringIO(stdout).readlines()
       diff = difflib.unified_diff(code, formatted_code,
diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py
index 82d3c19683..da7d74bdc4 100755
--- a/contrib/devtools/copyright_header.py
+++ b/contrib/devtools/copyright_header.py
@@ -146,7 +146,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name):
 ################################################################################
 
 def read_file(filename):
-    return open(os.path.abspath(filename), 'r').read()
+    return open(os.path.abspath(filename), 'r', encoding="utf8").read()
 
 def gather_file_info(filename):
     info = {}
@@ -325,13 +325,13 @@ def get_most_recent_git_change_year(filename):
 ################################################################################
 
 def read_file_lines(filename):
-    f = open(os.path.abspath(filename), 'r')
+    f = open(os.path.abspath(filename), 'r', encoding="utf8")
     file_lines = f.readlines()
     f.close()
     return file_lines
 
 def write_file_lines(filename, file_lines):
-    f = open(os.path.abspath(filename), 'w')
+    f = open(os.path.abspath(filename), 'w', encoding="utf8")
     f.write(''.join(file_lines))
     f.close()
 
diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py
index 187ef75fb7..4e90f85f50 100755
--- a/contrib/devtools/github-merge.py
+++ b/contrib/devtools/github-merge.py
@@ -191,7 +191,7 @@ def main():
     merge_branch = 'pull/'+pull+'/merge'
     local_merge_branch = 'pull/'+pull+'/local-merge'
 
-    devnull = open(os.devnull,'w')
+    devnull = open(os.devnull, 'w', encoding="utf8")
     try:
         subprocess.check_call([GIT,'checkout','-q',branch])
     except subprocess.CalledProcessError:
diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py
index 307e057736..9b6d6bf665 100755
--- a/contrib/devtools/test-security-check.py
+++ b/contrib/devtools/test-security-check.py
@@ -9,7 +9,7 @@ import subprocess
 import unittest
 
 def write_testcode(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding="utf8") as f:
         f.write('''
     #include <stdio.h>
     int main()
diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py
index 299377d691..df1db76e92 100755
--- a/contrib/filter-lcov.py
+++ b/contrib/filter-lcov.py
@@ -13,8 +13,8 @@ pattern = args.pattern
 outfile = args.outfile
 
 in_remove = False
-with open(tracefile, 'r') as f:
-    with open(outfile, 'w') as wf:
+with open(tracefile, 'r', encoding="utf8") as f:
+    with open(outfile, 'w', encoding="utf8") as wf:
         for line in f:
             for p in pattern:
                 if line.startswith("SF:") and p in line:
diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py
index f8aea27342..b501388fd2 100755
--- a/contrib/linearize/linearize-data.py
+++ b/contrib/linearize/linearize-data.py
@@ -75,7 +75,7 @@ def get_blk_dt(blk_hdr):
 # When getting the list of block hashes, undo any byte reversals.
 def get_block_hashes(settings):
     blkindex = []
-    f = open(settings['hashlist'], "r")
+    f = open(settings['hashlist'], "r", encoding="utf8")
     for line in f:
         line = line.rstrip()
         if settings['rev_hash_bytes'] == 'true':
@@ -261,7 +261,7 @@ if __name__ == '__main__':
         print("Usage: linearize-data.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1])
+    f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
         m = re.search('^\s*#', line)
diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py
index 8e1266ae0b..bfd2171947 100755
--- a/contrib/linearize/linearize-hashes.py
+++ b/contrib/linearize/linearize-hashes.py
@@ -96,7 +96,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
 
 def get_rpc_cookie():
     # Open the cookie file
-    with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f:
+    with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f:
         combined = f.readline()
         combined_split = combined.split(":")
         settings['rpcuser'] = combined_split[0]
@@ -107,7 +107,7 @@ if __name__ == '__main__':
         print("Usage: linearize-hashes.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1])
+    f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
         m = re.search('^\s*#', line)
diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py
index ace7d3534f..fe7cd1d597 100755
--- a/contrib/seeds/generate-seeds.py
+++ b/contrib/seeds/generate-seeds.py
@@ -127,10 +127,10 @@ def main():
     g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
     g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n')
     g.write(' */\n')
-    with open(os.path.join(indir,'nodes_main.txt'),'r') as f:
+    with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f:
         process_nodes(g, f, 'pnSeed6_main', 8333)
     g.write('\n')
-    with open(os.path.join(indir,'nodes_test.txt'),'r') as f:
+    with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f:
         process_nodes(g, f, 'pnSeed6_test', 18333)
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
 
diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py
index 80f0aa0bf1..a9e4977715 100755
--- a/contrib/verify-commits/verify-commits.py
+++ b/contrib/verify-commits/verify-commits.py
@@ -76,11 +76,11 @@ def main():
     # get directory of this program and read data files
     dirname = os.path.dirname(os.path.abspath(__file__))
     print("Using verify-commits data from " + dirname)
-    verified_root = open(dirname + "/trusted-git-root", "r").read().splitlines()[0]
-    verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r").read().splitlines()[0]
-    revsig_allowed = open(dirname + "/allow-revsig-commits", "r").read().splitlines()
-    unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r").read().splitlines()
-    incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r").read().splitlines()
+    verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0]
+    verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0]
+    revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines()
+    unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines()
+    incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines()
 
     # Set commit and branch and set variables
     current_commit = args.commit
diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py
index e8f0820ca8..e908071321 100755
--- a/share/qt/extract_strings_qt.py
+++ b/share/qt/extract_strings_qt.py
@@ -63,7 +63,7 @@ child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE)
 
 messages = parse_po(out.decode('utf-8'))
 
-f = open(OUT_CPP, 'w')
+f = open(OUT_CPP, 'w', encoding="utf8")
 f.write("""
 
 #include <QtGlobal>
diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py
index 8964c8d64b..6d51f31e35 100755
--- a/test/functional/feature_notifications.py
+++ b/test/functional/feature_notifications.py
@@ -36,7 +36,7 @@ class NotificationsTest(BitcoinTestFramework):
         wait_until(lambda: os.path.isfile(self.block_filename) and os.stat(self.block_filename).st_size >= (block_count * 65), timeout=10)
 
         # file content should equal the generated blocks hashes
-        with open(self.block_filename, 'r') as f:
+        with open(self.block_filename, 'r', encoding="utf-8") as f:
             assert_equal(sorted(blocks), sorted(l.strip() for l in f.read().splitlines()))
 
         self.log.info("test -walletnotify")
@@ -45,7 +45,7 @@ class NotificationsTest(BitcoinTestFramework):
 
         # file content should equal the generated transaction hashes
         txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count)))
-        with open(self.tx_filename, 'r') as f:
+        with open(self.tx_filename, 'r', encoding="ascii") as f:
             assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines()))
         os.remove(self.tx_filename)
 
@@ -58,7 +58,7 @@ class NotificationsTest(BitcoinTestFramework):
 
         # file content should equal the generated transaction hashes
         txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count)))
-        with open(self.tx_filename, 'r') as f:
+        with open(self.tx_filename, 'r', encoding="ascii") as f:
             assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines()))
 
         # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st.
diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py
index 060a2373b1..f573faaf17 100755
--- a/test/functional/rpc_getblockstats.py
+++ b/test/functional/rpc_getblockstats.py
@@ -81,11 +81,11 @@ class GetblockstatsTest(BitcoinTestFramework):
             'mocktime': int(mocktime),
             'stats': self.expected_stats,
         }
-        with open(filename, 'w') as f:
+        with open(filename, 'w', encoding="utf8") as f:
             json.dump(to_dump, f, sort_keys=True, indent=2)
 
     def load_test_data(self, filename):
-        with open(filename, 'r') as f:
+        with open(filename, 'r', encoding="utf8") as f:
             d = json.load(f)
             blocks = d['blocks']
             mocktime = d['mocktime']
diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py
index e016148f70..5e0b61b5e7 100644
--- a/test/functional/test_framework/util.py
+++ b/test/functional/test_framework/util.py
@@ -327,7 +327,7 @@ def get_auth_cookie(datadir):
                     assert password is None  # Ensure that there is only one rpcpassword line
                     password = line.split("=")[1].strip("\n")
     if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")):
-        with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f:
+        with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f:
             userpass = f.read()
             split_userpass = userpass.split(':')
             user = split_userpass[0]
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 5b3a4df0f9..36101d9f57 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -213,7 +213,7 @@ def main():
     # Read config generated by configure.
     config = configparser.ConfigParser()
     configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
-    config.read_file(open(configfile))
+    config.read_file(open(configfile, encoding="utf8"))
 
     passon_args.append("--configfile=%s" % configfile)
 
@@ -590,7 +590,7 @@ class RPCCoverage():
         if not os.path.isfile(coverage_ref_filename):
             raise RuntimeError("No coverage reference found")
 
-        with open(coverage_ref_filename, 'r') as coverage_ref_file:
+        with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file:
             all_cmds.update([line.strip() for line in coverage_ref_file.readlines()])
 
         for root, dirs, files in os.walk(self.dir):
@@ -599,7 +599,7 @@ class RPCCoverage():
                     coverage_filenames.add(os.path.join(root, filename))
 
         for filename in coverage_filenames:
-            with open(filename, 'r') as coverage_file:
+            with open(filename, 'r', encoding="utf8") as coverage_file:
                 covered_cmds.update([line.strip() for line in coverage_file.readlines()])
 
         return all_cmds - covered_cmds
diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py
index 53638615f6..a0fbc4a754 100755
--- a/test/functional/wallet_multiwallet.py
+++ b/test/functional/wallet_multiwallet.py
@@ -88,7 +88,7 @@ class MultiWalletTest(BitcoinTestFramework):
         self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
         # should not initialize if the specified walletdir is not a directory
         not_a_dir = wallet_dir('notadir')
-        open(not_a_dir, 'a').close()
+        open(not_a_dir, 'a', encoding="utf8").close()
         self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory')
 
         self.log.info("Do not allow -zapwallettxes with multiwallet")
diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py
index 7e96852c5c..c3cdeef580 100755
--- a/test/lint/check-rpc-mappings.py
+++ b/test/lint/check-rpc-mappings.py
@@ -44,7 +44,7 @@ def process_commands(fname):
     """Find and parse dispatch table in implementation file `fname`."""
     cmds = []
     in_rpcs = False
-    with open(fname, "r") as f:
+    with open(fname, "r", encoding="utf8") as f:
         for line in f:
             line = line.rstrip()
             if not in_rpcs:
@@ -70,7 +70,7 @@ def process_mapping(fname):
     """Find and parse conversion table in implementation file `fname`."""
     cmds = []
     in_rpcs = False
-    with open(fname, "r") as f:
+    with open(fname, "r", encoding="utf8") as f:
         for line in f:
             line = line.rstrip()
             if not in_rpcs:
diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh
new file mode 100755
index 0000000000..ce973e710a
--- /dev/null
+++ b/test/lint/lint-python-utf8-encoding.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to
+# avoid potential issues on the BSDs where the locale is not always set.
+
+EXIT_CODE=0
+OUTPUT=$(git grep " open(" -- "*.py" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]")
+if [[ ${OUTPUT} != "" ]]; then
+    echo "Python's open(...) seems to be used to open text files without explicitly"
+    echo "specifying encoding=\"utf8\":"
+    echo
+    echo "${OUTPUT}"
+    EXIT_CODE=1
+fi
+exit ${EXIT_CODE}
diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py
index 30bd13d0dc..16371a6234 100755
--- a/test/util/bitcoin-util-test.py
+++ b/test/util/bitcoin-util-test.py
@@ -28,7 +28,7 @@ import sys
 def main():
     config = configparser.ConfigParser()
     config.optionxform = str
-    config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini")))
+    config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8"))
     env_conf = dict(config.items('environment'))
 
     parser = argparse.ArgumentParser(description=__doc__)
@@ -49,7 +49,7 @@ def main():
 def bctester(testDir, input_basename, buildenv):
     """ Loads and parses the input file, runs all tests and reports results"""
     input_filename = os.path.join(testDir, input_basename)
-    raw_data = open(input_filename).read()
+    raw_data = open(input_filename, encoding="utf8").read()
     input_data = json.loads(raw_data)
 
     failed_testcases = []
@@ -86,7 +86,7 @@ def bctest(testDir, testObj, buildenv):
     inputData = None
     if "input" in testObj:
         filename = os.path.join(testDir, testObj["input"])
-        inputData = open(filename).read()
+        inputData = open(filename, encoding="utf8").read()
         stdinCfg = subprocess.PIPE
 
     # Read the expected output data (if there is any)
@@ -97,7 +97,7 @@ def bctest(testDir, testObj, buildenv):
         outputFn = testObj['output_cmp']
         outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
         try:
-            outputData = open(os.path.join(testDir, outputFn)).read()
+            outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read()
         except:
             logging.error("Output file " + outputFn + " can not be opened")
             raise
diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py
index 2456feb102..46e9fbc739 100755
--- a/test/util/rpcauth-test.py
+++ b/test/util/rpcauth-test.py
@@ -18,7 +18,7 @@ class TestRPCAuth(unittest.TestCase):
         config_path = os.path.abspath(
             os.path.join(os.sep, os.path.abspath(os.path.dirname(__file__)),
             "../config.ini"))
-        with open(config_path) as config_file:
+        with open(config_path, encoding="utf8") as config_file:
             config.read_file(config_file)
         sys.path.insert(0, os.path.dirname(config['environment']['RPCAUTH']))
         self.rpcauth = importlib.import_module('rpcauth')