Merge #13281: test: Move linters to test/lint, add readme

fa3c910bfeab00703c947c5200a64c21225b50ef test: Move linters to test/lint, add readme (MarcoFalke)

Pull request description:

  This moves the checks and linters from `devtools` to a subfolder in `test`. (Motivated by my opinion that the dev tools are mostly for generating code and updating the repo whereas the linters are read-only checks.)

  Also, adds a readme to clarify that checks and linters are only meant to prevent bugs and user facing issues, not merely stylistic preference or inconsistencies. (This is motivated by the diversity in developers and work flows as well as existing code styles. It would be too disruptive to change all existing code to a single style or too burdensome to force all developers to adhere to a single style. Also note that our style guide is changing, so locking in at the wrong style "too early" would only waste resources.)

Tree-SHA512: 9b10e89f2aeaf0c8a9ae248aa891d74e0abf0569f8e5dfd266446efa8bfaf19f0ea0980abf0b0b22f0d8416ee90d7435d21a9f9285b66df43f370b7979173406

15 files changed, 748 insertions, 0 deletions

diff --git a/test/functional/README.md b/test/functional/README.md
index 21050cc2fa..fdd7c339c5 100644
--- a/test/functional/README.md
+++ b/test/functional/README.md
@@ -20,6 +20,8 @@ don't have test cases for.
 - Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/)
 - Use a python linter like flake8 before submitting PRs to catch common style
   nits (eg trailing whitespace, unused imports, etc)
+- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that
+  could lead to bugs and issues in the test code.
 - Avoid wildcard imports where possible
 - Use a module-level docstring to describe what the test is testing, and how it
   is testing it.
diff --git a/test/lint/README.md b/test/lint/README.md
new file mode 100644
index 0000000000..15974a3598
--- /dev/null
+++ b/test/lint/README.md
@@ -0,0 +1,29 @@
+This folder contains lint scripts.
+
+check-doc.py
+============
+Check for missing documentation of command line options.
+
+commit-script-check.sh
+======================
+Verification of [scripted diffs](/doc/developer-notes.md#scripted-diffs).
+
+git-subtree-check.sh
+====================
+Run this script from the root of the repository to verify that a subtree matches the contents of
+the commit it claims to have been updated to.
+
+To use, make sure that you have fetched the upstream repository branch in which the subtree is
+maintained:
+* for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master)
+* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork)
+* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master)
+* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master)
+
+Usage: `git-subtree-check.sh DIR (COMMIT)`
+
+`COMMIT` may be omitted, in which case `HEAD` is used.
+
+lint-all.sh
+===========
+Calls other scripts with the `lint-` prefix.
diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py
new file mode 100755
index 0000000000..de5719eb29
--- /dev/null
+++ b/test/lint/check-doc.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+'''
+This checks if all command line args are documented.
+Return value is 0 to indicate no error.
+
+Author: @MarcoFalke
+'''
+
+from subprocess import check_output
+import re
+import sys
+
+FOLDER_GREP = 'src'
+FOLDER_TEST = 'src/test/'
+REGEX_ARG = '(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
+REGEX_DOC = 'AddArg\("(-[^"=]+?)(?:=|")'
+CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/{}'.format(FOLDER_GREP)
+CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST)
+CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR)
+# list unsupported, deprecated and duplicate args as they need no documentation
+SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-dbcrashratio', '-forcecompactdb', '-usehd'])
+
+
+def main():
+    used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True)
+    docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True)
+
+    args_used = set(re.findall(re.compile(REGEX_ARG), used))
+    args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)
+    args_need_doc = args_used.difference(args_docd)
+    args_unknown = args_docd.difference(args_used)
+
+    print("Args used        : {}".format(len(args_used)))
+    print("Args documented  : {}".format(len(args_docd)))
+    print("Args undocumented: {}".format(len(args_need_doc)))
+    print(args_need_doc)
+    print("Args unknown     : {}".format(len(args_unknown)))
+    print(args_unknown)
+
+    sys.exit(len(args_need_doc))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py
new file mode 100755
index 0000000000..7e96852c5c
--- /dev/null
+++ b/test/lint/check-rpc-mappings.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Check RPC argument consistency."""
+
+from collections import defaultdict
+import os
+import re
+import sys
+
+# Source files (relative to root) to scan for dispatch tables
+SOURCES = [
+    "src/rpc/server.cpp",
+    "src/rpc/blockchain.cpp",
+    "src/rpc/mining.cpp",
+    "src/rpc/misc.cpp",
+    "src/rpc/net.cpp",
+    "src/rpc/rawtransaction.cpp",
+    "src/wallet/rpcwallet.cpp",
+]
+# Source file (relative to root) containing conversion mapping
+SOURCE_CLIENT = 'src/rpc/client.cpp'
+# Argument names that should be ignored in consistency checks
+IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'}
+
+class RPCCommand:
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+class RPCArgument:
+    def __init__(self, names, idx):
+        self.names = names
+        self.idx = idx
+        self.convert = False
+
+def parse_string(s):
+    assert s[0] == '"'
+    assert s[-1] == '"'
+    return s[1:-1]
+
+def process_commands(fname):
+    """Find and parse dispatch table in implementation file `fname`."""
+    cmds = []
+    in_rpcs = False
+    with open(fname, "r") as f:
+        for line in f:
+            line = line.rstrip()
+            if not in_rpcs:
+                if re.match("static const CRPCCommand .*\[\] =", line):
+                    in_rpcs = True
+            else:
+                if line.startswith('};'):
+                    in_rpcs = False
+                elif '{' in line and '"' in line:
+                    m = re.search('{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
+                    assert m, 'No match to table expression: %s' % line
+                    name = parse_string(m.group(2))
+                    args_str = m.group(4).strip()
+                    if args_str:
+                        args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))]
+                    else:
+                        args = []
+                    cmds.append(RPCCommand(name, args))
+    assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps"
+    return cmds
+
+def process_mapping(fname):
+    """Find and parse conversion table in implementation file `fname`."""
+    cmds = []
+    in_rpcs = False
+    with open(fname, "r") as f:
+        for line in f:
+            line = line.rstrip()
+            if not in_rpcs:
+                if line == 'static const CRPCConvertParam vRPCConvertParams[] =':
+                    in_rpcs = True
+            else:
+                if line.startswith('};'):
+                    in_rpcs = False
+                elif '{' in line and '"' in line:
+                    m = re.search('{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
+                    assert m, 'No match to table expression: %s' % line
+                    name = parse_string(m.group(1))
+                    idx = int(m.group(2))
+                    argname = parse_string(m.group(3))
+                    cmds.append((name, idx, argname))
+    assert not in_rpcs and cmds
+    return cmds
+
+def main():
+    root = sys.argv[1]
+
+    # Get all commands from dispatch tables
+    cmds = []
+    for fname in SOURCES:
+        cmds += process_commands(os.path.join(root, fname))
+
+    cmds_by_name = {}
+    for cmd in cmds:
+        cmds_by_name[cmd.name] = cmd
+
+    # Get current convert mapping for client
+    client = SOURCE_CLIENT
+    mapping = set(process_mapping(os.path.join(root, client)))
+
+    print('* Checking consistency between dispatch tables and vRPCConvertParams')
+
+    # Check mapping consistency
+    errors = 0
+    for (cmdname, argidx, argname) in mapping:
+        try:
+            rargnames = cmds_by_name[cmdname].args[argidx].names
+        except IndexError:
+            print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname))
+            errors += 1
+            continue
+        if argname not in rargnames:
+            print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr)
+            errors += 1
+
+    # Check for conflicts in vRPCConvertParams conversion
+    # All aliases for an argument must either be present in the
+    # conversion table, or not. Anything in between means an oversight
+    # and some aliases won't work.
+    for cmd in cmds:
+        for arg in cmd.args:
+            convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names]
+            if any(convert) != all(convert):
+                print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert))
+                errors += 1
+            arg.convert = all(convert)
+
+    # Check for conversion difference by argument name.
+    # It is preferable for API consistency that arguments with the same name
+    # have the same conversion, so bin by argument name.
+    all_methods_by_argname = defaultdict(list)
+    converts_by_argname = defaultdict(list)
+    for cmd in cmds:
+        for arg in cmd.args:
+            for argname in arg.names:
+                all_methods_by_argname[argname].append(cmd.name)
+                converts_by_argname[argname].append(arg.convert)
+
+    for argname, convert in converts_by_argname.items():
+        if all(convert) != any(convert):
+            if argname in IGNORE_DUMMY_ARGS:
+                # these are testing or dummy, don't warn for them
+                continue
+            print('WARNING: conversion mismatch for argument named %s (%s)' %
+                  (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname]))))
+
+    sys.exit(errors > 0)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh
new file mode 100755
index 0000000000..1c9dbc7f68
--- /dev/null
+++ b/test/lint/commit-script-check.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# This simple script checks for commits beginning with: scripted-diff:
+# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and
+# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the
+# commit message.
+
+# The resulting script should exactly transform the previous commit into the current
+# one. Any remaining diff signals an error.
+
+if test "x$1" = "x"; then
+    echo "Usage: $0 <commit>..."
+    exit 1
+fi
+
+RET=0
+PREV_BRANCH=`git name-rev --name-only HEAD`
+PREV_HEAD=`git rev-parse HEAD`
+for i in `git rev-list --reverse $1`; do
+    if git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:"; then
+        git checkout --quiet $i^ || exit
+        SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`"
+        if test "x$SCRIPT" = "x"; then
+            echo "Error: missing script for: $i"
+            echo "Failed"
+            RET=1
+        else
+            echo "Running script for: $i"
+            echo "$SCRIPT"
+            eval "$SCRIPT"
+            git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1
+        fi
+        git reset --quiet --hard HEAD
+     else
+        if git rev-list "--format=%b" -n1 $i | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then
+            echo "Error: script block marker but no scripted-diff in title"
+            echo "Failed"
+            RET=1
+        fi
+    fi
+done
+git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD
+exit $RET
diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh
new file mode 100755
index 0000000000..184951715e
--- /dev/null
+++ b/test/lint/git-subtree-check.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+# Copyright (c) 2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+DIR="$1"
+COMMIT="$2"
+if [ -z "$COMMIT" ]; then
+    COMMIT=HEAD
+fi
+
+# Taken from git-subtree (Copyright (C) 2009 Avery Pennarun <apenwarr@gmail.com>)
+find_latest_squash()
+{
+	dir="$1"
+	sq=
+	main=
+	sub=
+	git log --grep="^git-subtree-dir: $dir/*\$" \
+		--pretty=format:'START %H%n%s%n%n%b%nEND%n' "$COMMIT" |
+	while read a b _; do
+		case "$a" in
+			START) sq="$b" ;;
+			git-subtree-mainline:) main="$b" ;;
+			git-subtree-split:) sub="$b" ;;
+			END)
+				if [ -n "$sub" ]; then
+					if [ -n "$main" ]; then
+						# a rejoin commit?
+						# Pretend its sub was a squash.
+						sq="$sub"
+					fi
+					echo "$sq" "$sub"
+					break
+				fi
+				sq=
+				main=
+				sub=
+				;;
+		esac
+	done
+}
+
+# find latest subtree update
+latest_squash="$(find_latest_squash "$DIR")"
+if [ -z "$latest_squash" ]; then
+    echo "ERROR: $DIR is not a subtree" >&2
+    exit 2
+fi
+set $latest_squash
+old=$1
+rev=$2
+
+# get the tree in the current commit
+tree_actual=$(git ls-tree -d "$COMMIT" "$DIR" | head -n 1)
+if [ -z "$tree_actual" ]; then
+    echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2
+    exit 1
+fi
+set $tree_actual
+tree_actual_type=$2
+tree_actual_tree=$3
+echo "$DIR in $COMMIT currently refers to $tree_actual_type $tree_actual_tree"
+if [ "d$tree_actual_type" != "dtree" ]; then
+    echo "FAIL: subtree directory $DIR is not a tree in $COMMIT" >&2
+    exit 1
+fi
+
+# get the tree at the time of the last subtree update
+tree_commit=$(git show -s --format="%T" $old)
+echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)"
+
+# ... and compare the actual tree with it
+if [ "$tree_actual_tree" != "$tree_commit" ]; then
+    git diff $tree_commit $tree_actual_tree >&2
+    echo "FAIL: subtree directory was touched without subtree merge" >&2
+    exit 1
+fi
+
+# get the tree in the subtree commit referred to
+if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then
+    echo "subtree commit $rev unavailable: cannot compare" >&2
+    exit
+fi
+tree_subtree=$(git show -s --format="%T" $rev)
+echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)"
+
+# ... and compare the actual tree with it
+if [ "$tree_actual_tree" != "$tree_subtree" ]; then
+    echo "FAIL: subtree update commit differs from upstream tree!" >&2
+    exit 1
+fi
+
+echo "GOOD"
diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh
new file mode 100755
index 0000000000..b6d86959c6
--- /dev/null
+++ b/test/lint/lint-all.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit
+# with a non-zero status code.
+
+set -u
+
+SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
+LINTALL=$(basename "${BASH_SOURCE[0]}")
+
+for f in "${SCRIPTDIR}"/lint-*.sh; do
+  if [ "$(basename "$f")" != "$LINTALL" ]; then
+    if ! "$f"; then
+      echo "^---- failure generated from $f"
+      exit 1
+    fi
+  fi
+done
diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh
new file mode 100755
index 0000000000..6a0dd556bb
--- /dev/null
+++ b/test/lint/lint-include-guards.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check include guards.
+
+HEADER_ID_PREFIX="BITCOIN_"
+HEADER_ID_SUFFIX="_H"
+
+REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)"
+
+EXIT_CODE=0
+for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}")
+do
+    HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr "[:lower:]" "[:upper:]")
+    HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}"
+    if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then
+        echo "${HEADER_FILE} seems to be missing the expected include guard:"
+        echo "  #ifndef ${HEADER_ID}"
+        echo "  #define ${HEADER_ID}"
+        echo "  ..."
+        echo "  #endif // ${HEADER_ID}"
+        echo
+        EXIT_CODE=1
+    fi
+done
+exit ${EXIT_CODE}
diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh
new file mode 100755
index 0000000000..f54be46b52
--- /dev/null
+++ b/test/lint/lint-includes.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check for duplicate includes.
+
+filter_suffix() {
+    git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "/(leveldb|secp256k1|univalue)/"
+}
+
+EXIT_CODE=0
+for HEADER_FILE in $(filter_suffix h); do
+    DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d)
+    if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then
+        echo "Duplicate include(s) in ${HEADER_FILE}:"
+        echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}"
+        echo
+        EXIT_CODE=1
+    fi
+done
+for CPP_FILE in $(filter_suffix cpp); do
+    DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d)
+    if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then
+        echo "Duplicate include(s) in ${CPP_FILE}:"
+        echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}"
+        echo
+        EXIT_CODE=1
+    fi
+done
+exit ${EXIT_CODE}
diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh
new file mode 100755
index 0000000000..35be13ec19
--- /dev/null
+++ b/test/lint/lint-logs.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check that all logs are terminated with '\n'
+#
+# Some logs are continued over multiple lines. They should be explicitly
+# commented with \* Continued *\
+#
+# There are some instances of LogPrintf() in comments. Those can be
+# ignored
+
+
+UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \
+    grep -v '\\n"' | \
+    grep -v "/\* Continued \*/" | \
+    grep -v "LogPrint()" | \
+    grep -v "LogPrintf()")
+if [[ ${UNTERMINATED_LOGS} != "" ]]; then
+    echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n"
+    echo
+    echo "${UNTERMINATED_LOGS}"
+    exit 1
+fi
diff --git a/test/lint/lint-python-shebang.sh b/test/lint/lint-python-shebang.sh
new file mode 100755
index 0000000000..f5c5971c03
--- /dev/null
+++ b/test/lint/lint-python-shebang.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+# Shebang must use python3 (not python or python2)
+EXIT_CODE=0
+for PYTHON_FILE in $(git ls-files -- "*.py"); do
+    if [[ $(head -c 2 "${PYTHON_FILE}") == "#!" &&
+          $(head -n 1 "${PYTHON_FILE}") != "#!/usr/bin/env python3" ]]; then
+        echo "Missing shebang \"#!/usr/bin/env python3\" in ${PYTHON_FILE} (do not use python or python2)"
+        EXIT_CODE=1
+    fi
+done
+exit ${EXIT_CODE}
diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh
new file mode 100755
index 0000000000..7d3555b6d4
--- /dev/null
+++ b/test/lint/lint-python.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check for specified flake8 warnings in python files.
+
+# E101 indentation contains mixed spaces and tabs
+# E112 expected an indented block
+# E113 unexpected indentation
+# E115 expected an indented block (comment)
+# E116 unexpected indentation (comment)
+# E125 continuation line with same indent as next logical line
+# E129 visually indented line with same indent as next logical line
+# E131 continuation line unaligned for hanging indent
+# E133 closing bracket is missing indentation
+# E223 tab before operator
+# E224 tab after operator
+# E242 tab after ','
+# E266 too many leading '#' for block comment
+# E271 multiple spaces after keyword
+# E272 multiple spaces before keyword
+# E273 tab after keyword
+# E274 tab before keyword
+# E275 missing whitespace after keyword
+# E304 blank lines found after function decorator
+# E306 expected 1 blank line before a nested definition
+# E401 multiple imports on one line
+# E402 module level import not at top of file
+# E502 the backslash is redundant between brackets
+# E701 multiple statements on one line (colon)
+# E702 multiple statements on one line (semicolon)
+# E703 statement ends with a semicolon
+# E714 test for object identity should be "is not"
+# E721 do not compare types, use "isinstance()"
+# E741 do not use variables named "l", "O", or "I"
+# E742 do not define classes named "l", "O", or "I"
+# E743 do not define functions named "l", "O", or "I"
+# E901 SyntaxError: invalid syntax
+# E902 TokenError: EOF in multi-line string
+# F401 module imported but unused
+# F402 import module from line N shadowed by loop variable
+# F404 future import(s) name after other statements
+# F406 "from module import *" only allowed at module level
+# F407 an undefined __future__ feature name was imported
+# F601 dictionary key name repeated with different values
+# F602 dictionary key variable name repeated with different values
+# F621 too many expressions in an assignment with star-unpacking
+# F622 two or more starred expressions in an assignment (a, *b, *c = d)
+# F631 assertion test is a tuple, which are always True
+# F701 a break statement outside of a while or for loop
+# F702 a continue statement outside of a while or for loop
+# F703 a continue statement in a finally block in a loop
+# F704 a yield or yield from statement outside of a function
+# F705 a return statement with arguments inside a generator
+# F706 a return statement outside of a function/method
+# F707 an except: block as not the last exception handler
+# F811 redefinition of unused name from line N
+# F812 list comprehension redefines 'foo' from line N
+# F821 undefined name 'Foo'
+# F822 undefined name name in __all__
+# F823 local variable name … referenced before assignment
+# F831 duplicate argument name in function definition
+# F841 local variable 'foo' is assigned to but never used
+# W191 indentation contains tabs
+# W291 trailing whitespace
+# W292 no newline at end of file
+# W293 blank line contains whitespace
+# W504 line break after binary operator
+# W601 .has_key() is deprecated, use "in"
+# W602 deprecated form of raising exception
+# W603 "<>" is deprecated, use "!="
+# W604 backticks are deprecated, use "repr()"
+# W605 invalid escape sequence "x"
+# W606 'async' and 'await' are reserved keywords starting with Python 3.7
+
+flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 .
diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh
new file mode 100755
index 0000000000..5f5fa9a925
--- /dev/null
+++ b/test/lint/lint-shell.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check for shellcheck warnings in shell scripts.
+
+# Disabled warnings:
+# SC2001: See if you can use ${variable//search/replace} instead.
+# SC2004: $/${} is unnecessary on arithmetic variables.
+# SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'.
+# SC2006: Use $(..) instead of legacy `..`.
+# SC2016: Expressions don't expand in single quotes, use double quotes for that.
+# SC2028: echo won't expand escape sequences. Consider printf.
+# SC2046: Quote this to prevent word splitting.
+# SC2048: Use "$@" (with quotes) to prevent whitespace problems.
+# SC2066: Since you double quoted this, it will not word split, and the loop will only run once.
+# SC2086: Double quote to prevent globbing and word splitting.
+# SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'.
+# SC2148: Tips depend on target shell and yours is unknown. Add a shebang.
+# SC2162: read without -r will mangle backslashes.
+# SC2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined.
+# SC2166: Prefer [ p ] || [ q ] as [ p -o q ] is not well defined.
+# SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?.
+shellcheck -e SC2001,SC2004,SC2005,SC2006,SC2016,SC2028,SC2046,SC2048,SC2066,SC2086,SC2116,SC2148,SC2162,SC2166,SC2181 \
+    $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/')
diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh
new file mode 100755
index 0000000000..ffc0660551
--- /dev/null
+++ b/test/lint/lint-tests.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check the test suite naming conventions
+
+EXIT_CODE=0
+
+NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \
+    "src/test/**.cpp" "src/wallet/test/**.cpp" | \
+    grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$')
+if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then
+    echo "The test suite in file src/test/foo_tests.cpp should be named"
+    echo "\"foo_tests\". Please make sure the following test suites follow"
+    echo "that convention:"
+    echo
+    echo "${NAMING_INCONSISTENCIES}"
+    EXIT_CODE=1
+fi
+
+TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \
+    "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \
+    sort | uniq -d)
+if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then
+    echo "Test suite names must be unique. The following test suite names"
+    echo "appear to be used more than once:"
+    echo
+    echo "${TEST_SUITE_NAME_COLLISSIONS}"
+    EXIT_CODE=1
+fi
+
+exit ${EXIT_CODE}
diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh
new file mode 100755
index 0000000000..c5d43043d5
--- /dev/null
+++ b/test/lint/lint-whitespace.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Check for new lines in diff that introduce trailing whitespace.
+
+# We can't run this check unless we know the commit range for the PR.
+
+while getopts "?" opt; do
+  case $opt in
+    ?)
+      echo "Usage: .lint-whitespace.sh [N]"
+      echo "       TRAVIS_COMMIT_RANGE='<commit range>' .lint-whitespace.sh"
+      echo "       .lint-whitespace.sh -?"
+      echo "Checks unstaged changes, the previous N commits, or a commit range."
+      echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' .lint-whitespace.sh"
+      exit 0
+    ;;
+  esac
+done
+
+if [ -z "${TRAVIS_COMMIT_RANGE}" ]; then
+  if [ "$1" ]; then
+    TRAVIS_COMMIT_RANGE="HEAD~$1...HEAD"
+  else
+    TRAVIS_COMMIT_RANGE="HEAD"
+  fi
+fi
+
+showdiff() {
+  if ! git diff -U0 "${TRAVIS_COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)src/leveldb/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
+    echo "Failed to get a diff"
+    exit 1
+  fi
+}
+
+showcodediff() {
+  if ! git diff -U0 "${TRAVIS_COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
+    echo "Failed to get a diff"
+    exit 1
+  fi
+}
+
+RET=0
+
+# Check if trailing whitespace was found in the diff.
+if showdiff | grep -E -q '^\+.*\s+$'; then
+  echo "This diff appears to have added new lines with trailing whitespace."
+  echo "The following changes were suspected:"
+  FILENAME=""
+  SEEN=0
+  SEENLN=0
+  while read -r line; do
+    if [[ "$line" =~ ^diff ]]; then
+      FILENAME="$line"
+      SEEN=0
+    elif [[ "$line" =~ ^@@ ]]; then
+      LINENUMBER="$line"
+      SEENLN=0
+    else
+      if [ "$SEEN" -eq 0 ]; then
+        # The first time a file is seen with trailing whitespace, we print the
+        # filename (preceded by a newline).
+        echo
+        echo "$FILENAME"
+        SEEN=1
+      fi
+      if [ "$SEENLN" -eq 0 ]; then
+        echo "$LINENUMBER"
+        SEENLN=1
+      fi
+      echo "$line"
+    fi
+  done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)')
+  RET=1
+fi
+
+# Check if tab characters were found in the diff.
+if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then
+  echo "This diff appears to have added new lines with tab characters instead of spaces."
+  echo "The following changes were suspected:"
+  FILENAME=""
+  SEEN=0
+  SEENLN=0
+  while read -r line; do
+    if [[ "$line" =~ ^diff ]]; then
+      FILENAME="$line"
+      SEEN=0
+    elif [[ "$line" =~ ^@@ ]]; then
+      LINENUMBER="$line"
+      SEENLN=0
+    else
+      if [ "$SEEN" -eq 0 ]; then
+        # The first time a file is seen with a tab character, we print the
+        # filename (preceded by a newline).
+        echo
+        echo "$FILENAME"
+        SEEN=1
+      fi
+      if [ "$SEENLN" -eq 0 ]; then
+        echo "$LINENUMBER"
+        SEENLN=1
+      fi
+      echo "$line"
+    fi
+  done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}')
+  RET=1
+fi
+
+exit $RET