7 files changed, 350 insertions, 28 deletions
diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py
index b484bffe0d..10f8b92a8f 100755
--- a/test/functional/feature_cltv.py
+++ b/test/functional/feature_cltv.py
@@ -62,7 +62,7 @@ def create_transaction(node, coinbase, to_address, amount):
 class BIP65Test(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-        self.extra_args = [['-promiscuousmempoolflags=1', '-whitelist=127.0.0.1']]
+        self.extra_args = [['-whitelist=127.0.0.1']]
         self.setup_clean_chain = True
 
     def run_test(self):
@@ -116,12 +116,13 @@ class BIP65Test(BitcoinTestFramework):
         spendtx.rehash()
 
         # First we show that this tx is valid except for CLTV by getting it
-        # accepted to the mempool (which we can achieve with
-        # -promiscuousmempoolflags).
-        self.nodes[0].p2p.send_and_ping(msg_tx(spendtx))
-        assert spendtx.hash in self.nodes[0].getrawmempool()
+        # rejected from the mempool for exactly that reason.
+        assert_equal(
+            [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}],
+            self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True)
+        )
 
-        # Now we verify that a block with this transaction is invalid.
+        # Now we verify that a block with this transaction is also invalid.
         block.vtx.append(spendtx)
         block.hashMerkleRoot = block.calc_merkle_root()
         block.solve()
diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py
index 13224466d3..4337d42dc0 100755
--- a/test/functional/feature_dersig.py
+++ b/test/functional/feature_dersig.py
@@ -47,10 +47,11 @@ def create_transaction(node, coinbase, to_address, amount):
     tx.deserialize(BytesIO(hex_str_to_bytes(signresult['hex'])))
     return tx
 
+
 class BIP66Test(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-        self.extra_args = [['-promiscuousmempoolflags=1', '-whitelist=127.0.0.1']]
+        self.extra_args = [['-whitelist=127.0.0.1']]
         self.setup_clean_chain = True
 
     def run_test(self):
@@ -108,12 +109,13 @@ class BIP66Test(BitcoinTestFramework):
         spendtx.rehash()
 
         # First we show that this tx is valid except for DERSIG by getting it
-        # accepted to the mempool (which we can achieve with
-        # -promiscuousmempoolflags).
-        self.nodes[0].p2p.send_and_ping(msg_tx(spendtx))
-        assert spendtx.hash in self.nodes[0].getrawmempool()
+        # rejected from the mempool for exactly that reason.
+        assert_equal(
+            [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}],
+            self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True)
+        )
 
-        # Now we verify that a block with this transaction is invalid.
+        # Now we verify that a block with this transaction is also invalid.
         block.vtx.append(spendtx)
         block.hashMerkleRoot = block.calc_merkle_root()
         block.rehash()
diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py
index b10306b283..99ad2d5222 100755
--- a/test/functional/feature_segwit.py
+++ b/test/functional/feature_segwit.py
@@ -43,8 +43,8 @@ class SegWitTest(BitcoinTestFramework):
         self.num_nodes = 3
         # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation.
         self.extra_args = [["-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"],
-                           ["-blockversion=4", "-promiscuousmempoolflags=517", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"],
-                           ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"]]
+                           ["-blockversion=4", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"],
+                           ["-blockversion=536870915", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"]]
 
     def setup_network(self):
         super().setup_network()
@@ -64,12 +64,8 @@ class SegWitTest(BitcoinTestFramework):
         sync_blocks(self.nodes)
 
     def fail_accept(self, node, error_msg, txid, sign, redeem_script=""):
-        assert_raises_rpc_error(-26, error_msg, send_to_witness, 1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
+        assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script)
 
-    def fail_mine(self, node, txid, sign, redeem_script=""):
-        send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
-        assert_raises_rpc_error(-1, "CreateNewBlock: TestBlockValidity failed", node.generate, 1)
-        sync_blocks(self.nodes)
 
     def run_test(self):
         self.nodes[0].generate(161) #block 161
@@ -171,10 +167,10 @@ class SegWitTest(BitcoinTestFramework):
             assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness()))
 
         self.log.info("Verify witness txs without witness data are invalid after the fork")
-        self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][2], False)
-        self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][2], False)
-        self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][2], False, witness_script(False, self.pubkey[2]))
-        self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][2], False, witness_script(True, self.pubkey[2]))
+        self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False)
+        self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness) (code 64)', wit_ids[NODE_2][WIT_V1][2], sign=False)
+        self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', p2sh_ids[NODE_2][WIT_V0][2], sign=False, redeem_script=witness_script(False, self.pubkey[2]))
+        self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness) (code 64)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2]))
 
         self.log.info("Verify default node can now use witness txs")
         self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) #block 432
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index d324cd9bba..b5c440625f 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -229,7 +229,7 @@ def main():
     logging.basicConfig(format='%(message)s', level=logging_level)
 
     # Create base test directory
-    tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
+    tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
     os.makedirs(tmpdir)
 
     logging.debug("Temporary test directory at %s" % tmpdir)
diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py
index fa5a2154a4..e2938dba3b 100755
--- a/test/functional/wallet_multiwallet.py
+++ b/test/functional/wallet_multiwallet.py
@@ -30,6 +30,11 @@ class MultiWalletTest(BitcoinTestFramework):
         wallet_dir = lambda *p: data_dir('wallets', *p)
         wallet = lambda name: node.get_wallet_rpc(name)
 
+        def wallet_file(name):
+            if os.path.isdir(wallet_dir(name)):
+                return wallet_dir(name, "wallet.dat")
+            return wallet_dir(name)
+
         # check wallet.dat is created
         self.stop_nodes()
         assert_equal(os.path.isfile(wallet_dir('wallet.dat')), True)
@@ -43,6 +48,12 @@ class MultiWalletTest(BitcoinTestFramework):
         # directory paths) can be loaded
         os.rename(wallet_dir("wallet.dat"), wallet_dir("w8"))
 
+        # create another dummy wallet for use in testing backups later
+        self.start_node(0, [])
+        self.stop_nodes()
+        empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat')
+        os.rename(wallet_dir("wallet.dat"), empty_wallet)
+
         # restart node with a mix of wallet names:
         #   w1, w2, w3 - to verify new wallets created when non-existing paths specified
         #   w          - to verify wallet name matching works when one wallet path is prefix of another
@@ -59,10 +70,7 @@ class MultiWalletTest(BitcoinTestFramework):
         # check that all requested wallets were created
         self.stop_node(0)
         for wallet_name in wallet_names:
-            if os.path.isdir(wallet_dir(wallet_name)):
-                assert_equal(os.path.isfile(wallet_dir(wallet_name, "wallet.dat")), True)
-            else:
-                assert_equal(os.path.isfile(wallet_dir(wallet_name)), True)
+            assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
 
         # should not initialize if wallet path can't be created
         exp_stderr = "boost::filesystem::create_directory: (The system cannot find the path specified|Not a directory):"
@@ -265,5 +273,25 @@ class MultiWalletTest(BitcoinTestFramework):
         assert_equal(self.nodes[0].listwallets(), ['w1'])
         assert_equal(w1.getwalletinfo()['walletname'], 'w1')
 
+        # Test backing up and restoring wallets
+        self.log.info("Test wallet backup")
+        self.restart_node(0, ['-nowallet'])
+        for wallet_name in wallet_names:
+            self.nodes[0].loadwallet(wallet_name)
+        for wallet_name in wallet_names:
+            rpc = self.nodes[0].get_wallet_rpc(wallet_name)
+            addr = rpc.getnewaddress()
+            backup = os.path.join(self.options.tmpdir, 'backup.dat')
+            rpc.backupwallet(backup)
+            self.nodes[0].unloadwallet(wallet_name)
+            shutil.copyfile(empty_wallet, wallet_file(wallet_name))
+            self.nodes[0].loadwallet(wallet_name)
+            assert_equal(rpc.getaddressinfo(addr)['ismine'], False)
+            self.nodes[0].unloadwallet(wallet_name)
+            shutil.copyfile(backup, wallet_file(wallet_name))
+            self.nodes[0].loadwallet(wallet_name)
+            assert_equal(rpc.getaddressinfo(addr)['ismine'], True)
+
+
 if __name__ == '__main__':
     MultiWalletTest().main()
diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py
new file mode 100755
index 0000000000..c07dfe317b
--- /dev/null
+++ b/test/lint/lint-format-strings.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Lint format strings: This program checks that the number of arguments passed
+# to a variadic format string function matches the number of format specifiers
+# in the format string.
+
+import argparse
+import re
+import sys
+
+FALSE_POSITIVES = [
+    ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"),
+    ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"),
+    ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"),
+    ("src/util.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"),
+    ("src/util.cpp", "strprintf(COPYRIGHT_HOLDERS, COPYRIGHT_HOLDERS_SUBSTITUTION)"),
+    ("src/wallet/wallet.h",  "WalletLogPrintf(std::string fmt, Params... parameters)"),
+    ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"),
+]
+
+
+def parse_function_calls(function_name, source_code):
+    """Return an array with all calls to function function_name in string source_code.
+    Preprocessor directives and C++ style comments ("//") in source_code are removed.
+
+    >>> len(parse_function_calls("foo", "foo();bar();foo();bar();"))
+    2
+    >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);")
+    True
+    >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);")
+    True
+    >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();"))
+    1
+    >>> len(parse_function_calls("foo", "#define FOO foo();"))
+    0
+    """
+    assert(type(function_name) is str and type(source_code) is str and function_name)
+    lines = [re.sub("// .*", " ", line).strip()
+             for line in source_code.split("\n")
+             if not line.strip().startswith("#")]
+    return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines))
+
+
+def normalize(s):
+    """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */")
+    replaced with spaces. Multiple spaces are replaced with a single space.
+
+    >>> normalize("  /* nothing */   foo\tfoo  /* bar */  foo     ")
+    'foo foo foo'
+    """
+    assert(type(s) is str)
+    s = s.replace("\n", " ")
+    s = s.replace("\t", " ")
+    s = re.sub("/\*.*?\*/", " ", s)
+    s = re.sub(" {2,}", " ", s)
+    return s.strip()
+
+
+ESCAPE_MAP = {
+    r"\n": "[escaped-newline]",
+    r"\t": "[escaped-tab]",
+    r'\"': "[escaped-quote]",
+}
+
+
+def escape(s):
+    """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as
+    "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]".
+
+    >>> unescape(escape("foo")) == "foo"
+    True
+    >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"')
+    'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]'
+    """
+    assert(type(s) is str)
+    for raw_value, escaped_value in ESCAPE_MAP.items():
+        s = s.replace(raw_value, escaped_value)
+    return s
+
+
+def unescape(s):
+    """Return the unescaped version of escaped string s.
+    Reverses the replacements made in function escape(s).
+
+    >>> unescape(escape("bar"))
+    'bar'
+    >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]")
+    'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"'
+    """
+    assert(type(s) is str)
+    for raw_value, escaped_value in ESCAPE_MAP.items():
+        s = s.replace(escaped_value, raw_value)
+    return s
+
+
+def parse_function_call_and_arguments(function_name, function_call):
+    """Split string function_call into an array of strings consisting of:
+    * the string function_call followed by "("
+    * the function call argument #1
+    * ...
+    * the function call argument #n
+    * a trailing ");"
+
+    The strings returned are in escaped form. See escape(...).
+
+    >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
+    ['foo(', '"%s",', ' "foo"', ')']
+    >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
+    ['foo(', '"%s",', ' "foo"', ')']
+    >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");')
+    ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')']
+    >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);')
+    ['fooprintf(', '"%050d",', ' i', ')']
+    >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo')
+    ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')']
+    >>> parse_function_call_and_arguments("foo", "foo()")
+    ['foo(', '', ')']
+    >>> parse_function_call_and_arguments("foo", "foo(123)")
+    ['foo(', '123', ')']
+    >>> parse_function_call_and_arguments("foo", 'foo("foo")')
+    ['foo(', '"foo"', ')']
+    """
+    assert(type(function_name) is str and type(function_call) is str and function_name)
+    remaining = normalize(escape(function_call))
+    expected_function_call = "{}(".format(function_name)
+    assert(remaining.startswith(expected_function_call))
+    parts = [expected_function_call]
+    remaining = remaining[len(expected_function_call):]
+    open_parentheses = 1
+    in_string = False
+    parts.append("")
+    for char in remaining:
+        parts.append(parts.pop() + char)
+        if char == "\"":
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if char == "(":
+            open_parentheses += 1
+            continue
+        if char == ")":
+            open_parentheses -= 1
+        if open_parentheses > 1:
+            continue
+        if open_parentheses == 0:
+            parts.append(parts.pop()[:-1])
+            parts.append(char)
+            break
+        if char == ",":
+            parts.append("")
+    return parts
+
+
+def parse_string_content(argument):
+    """Return the text within quotes in string argument.
+
+    >>> parse_string_content('1 "foo %d bar" 2')
+    'foo %d bar'
+    >>> parse_string_content('1 foobar 2')
+    ''
+    >>> parse_string_content('1 "bar" 2')
+    'bar'
+    >>> parse_string_content('1 "foo" 2 "bar" 3')
+    'foobar'
+    >>> parse_string_content('1 "foo" 2 " " "bar" 3')
+    'foo bar'
+    >>> parse_string_content('""')
+    ''
+    >>> parse_string_content('')
+    ''
+    >>> parse_string_content('1 2 3')
+    ''
+    """
+    assert(type(argument) is str)
+    string_content = ""
+    in_string = False
+    for char in normalize(escape(argument)):
+        if char == "\"":
+            in_string = not in_string
+        elif in_string:
+            string_content += char
+    return string_content
+
+
+def count_format_specifiers(format_string):
+    """Return the number of format specifiers in string format_string.
+
+    >>> count_format_specifiers("foo bar foo")
+    0
+    >>> count_format_specifiers("foo %d bar foo")
+    1
+    >>> count_format_specifiers("foo %d bar %i foo")
+    2
+    >>> count_format_specifiers("foo %d bar %i foo %% foo")
+    2
+    >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo")
+    3
+    >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo")
+    4
+    """
+    assert(type(format_string) is str)
+    n = 0
+    in_specifier = False
+    for i, char in enumerate(format_string):
+        if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%":
+            pass
+        elif char == "%":
+            in_specifier = True
+            n += 1
+        elif char in "aAcdeEfFgGinopsuxX":
+            in_specifier = False
+        elif in_specifier and char == "*":
+            n += 1
+    return n
+
+
+def main():
+    parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
+                                     "to a variadic format string function matches the number of format "
+                                     "specifiers in the format string.")
+    parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string "
+                        "argument (e.g. 1 in the case of fprintf)", default=0)
+    parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
+    parser.add_argument("file", type=argparse.FileType("r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)")
+    args = parser.parse_args()
+
+    exit_code = 0
+    for f in args.file:
+        for function_call_str in parse_function_calls(args.function_name, f.read()):
+            parts = parse_function_call_and_arguments(args.function_name, function_call_str)
+            relevant_function_call_str = unescape("".join(parts))[:512]
+            if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
+                continue
+            if len(parts) < 3 + args.skip_arguments:
+                exit_code = 1
+                print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
+                continue
+            argument_count = len(parts) - 3 - args.skip_arguments
+            format_str = parse_string_content(parts[1 + args.skip_arguments])
+            format_specifier_count = count_format_specifiers(format_str)
+            if format_specifier_count != argument_count:
+                exit_code = 1
+                print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
+                continue
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh
new file mode 100755
index 0000000000..17f846d29b
--- /dev/null
+++ b/test/lint/lint-format-strings.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Lint format strings: This program checks that the number of arguments passed
+# to a variadic format string function matches the number of format specifiers
+# in the format string.
+
+export LC_ALL=C
+
+FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=(
+    FatalError,0
+    fprintf,1
+    LogConnectFailure,1
+    LogPrint,1
+    LogPrintf,0
+    printf,0
+    snprintf,2
+    sprintf,1
+    strprintf,0
+    vfprintf,1
+    vprintf,1
+    vsnprintf,1
+    vsprintf,1
+    WalletLogPrintf,0
+)
+
+EXIT_CODE=0
+if ! python3 -m doctest test/lint/lint-format-strings.py; then
+    EXIT_CODE=1
+fi
+for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
+    IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
+    mapfile -t MATCHING_FILES < <(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)")
+    if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
+        EXIT_CODE=1
+    fi
+done
+exit ${EXIT_CODE}