diff options
Diffstat (limited to 'test')
103 files changed, 3353 insertions, 1148 deletions
diff --git a/test/README.md b/test/README.md index 680f9bf9a6..9d4351b1de 100644 --- a/test/README.md +++ b/test/README.md @@ -18,7 +18,8 @@ request is opened. All sets of tests can also be run locally. # Running tests locally -Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. +Before tests can be run locally, Bitcoin Core must be built. See the [building instructions](/doc#building) for help. + ### Functional tests @@ -174,7 +175,28 @@ cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid gdb /home/example/bitcoind <pid> ``` -Note: gdb attach step may require `sudo` +Note: gdb attach step may require ptrace_scope to be modified, or `sudo` preceding the `gdb`. +See this link for considerations: https://www.kernel.org/doc/Documentation/security/Yama.txt + +##### Profiling + +An easy way to profile node performance during functional tests is provided +for Linux platforms using `perf`. + +Perf will sample the running node and will generate profile data in the node's +datadir. The profile data can then be presented using `perf report` or a graphical +tool like [hotspot](https://github.com/KDAB/hotspot). + +To generate a profile during test suite runs, use the `--perf` flag. + +To see render the output to text, run + +```sh +perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less +``` + +For ways to generate more granular profiles, see the README in +[test/functional](/test/functional). ### Util tests diff --git a/test/functional/README.md b/test/functional/README.md index d40052ac93..5e3009e6af 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -20,9 +20,13 @@ don't have test cases for. - Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/) - Use a python linter like flake8 before submitting PRs to catch common style nits (eg trailing whitespace, unused imports, etc) +- The oldest supported Python version is specified in [doc/dependencies.md](/doc/dependencies.md). + Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), + to prevent accidentally introducing modern syntax from an unsupported Python version. + The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that could lead to bugs and issues in the test code. -- Avoid wildcard imports where possible +- Avoid wildcard imports - Use a module-level docstring to describe what the test is testing, and how it is testing it. - When subclassing the BitcoinTestFramwork, place overrides for the @@ -39,6 +43,7 @@ don't have test cases for. - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` - use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` @@ -118,3 +123,36 @@ Helpers for script.py #### [test_framework/blocktools.py](test_framework/blocktools.py) Helper functions for creating blocks and transactions. + +### Benchmarking with perf + +An easy way to profile node performance during functional tests is provided +for Linux platforms using `perf`. + +Perf will sample the running node and will generate profile data in the node's +datadir. The profile data can then be presented using `perf report` or a graphical +tool like [hotspot](https://github.com/KDAB/hotspot). + +There are two ways of invoking perf: one is to use the `--perf` flag when +running tests, which will profile each node during the entire test run: perf +begins to profile when the node starts and ends when it shuts down. The other +way is the use the `profile_with_perf` context manager, e.g. + +```python +with node.profile_with_perf("send-big-msgs"): + # Perform activity on the node you're interested in profiling, e.g.: + for _ in range(10000): + node.p2p.send_message(some_large_message) +``` + +To see useful textual output, run + +```sh +perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less +``` + +#### See also: + +- [Installing perf](https://askubuntu.com/q/50145) +- [Perf examples](http://www.brendangregg.com/perf.html) +- [Hotspot](https://github.com/KDAB/hotspot): a GUI for perf output analysis diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 3230d5cb6b..5bb3b5c094 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -2,7 +2,9 @@ """Combine logs from multiple bitcoin nodes as well as the test_framework log. This streams the combined log output to stdout. Use combine_logs.py > outputfile -to write to an outputfile.""" +to write to an outputfile. + +If no argument is provided, the most recent test directory will be used.""" import argparse from collections import defaultdict, namedtuple @@ -11,6 +13,13 @@ import itertools import os import re import sys +import tempfile + +# N.B.: don't import any local modules here - this script must remain executable +# without the parent module installed. + +# Should match same symbol in `test_framework.test_framework`. +TMPDIR_PREFIX = "bitcoin_func_test_" # Matches on the date format at the start of the log event TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z") @@ -19,22 +28,30 @@ LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) def main(): """Main function. Parses args, reads the log files and renders them as text or html.""" - - parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + 'testdir', nargs='?', default='', + help=('temporary test directory to combine logs from. ' + 'Defaults to the most recent')) parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)') parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2') - args, unknown_args = parser.parse_known_args() + args = parser.parse_args() if args.html and args.color: print("Only one out of --color or --html should be specified") sys.exit(1) - # There should only be one unknown argument - the path of the temporary test directory - if len(unknown_args) != 1: - print("Unexpected arguments" + str(unknown_args)) + testdir = args.testdir or find_latest_test_dir() + + if not testdir: + print("No test directories found") sys.exit(1) - log_events = read_logs(unknown_args[0]) + if not args.testdir: + print("Opening latest test directory: {}".format(testdir), file=sys.stderr) + + log_events = read_logs(testdir) print_logs(log_events, color=args.color, html=args.html) @@ -53,6 +70,29 @@ def read_logs(tmp_dir): return heapq.merge(*[get_log_events(source, f) for source, f in files]) + +def find_latest_test_dir(): + """Returns the latest tmpfile test directory prefix.""" + tmpdir = tempfile.gettempdir() + + def join_tmp(basename): + return os.path.join(tmpdir, basename) + + def is_valid_test_tmpdir(basename): + fullpath = join_tmp(basename) + return ( + os.path.isdir(fullpath) + and basename.startswith(TMPDIR_PREFIX) + and os.access(fullpath, os.R_OK) + ) + + testdir_paths = [ + join_tmp(name) for name in os.listdir(tmpdir) if is_valid_test_tmpdir(name) + ] + + return max(testdir_paths, key=os.path.getmtime) if testdir_paths else None + + def get_log_events(source, logfile): """Generator function that returns individual log events. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py new file mode 100644 index 0000000000..02deae92f3 --- /dev/null +++ b/test/functional/data/invalid_txs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Templates for constructing various sorts of invalid transactions. + +These templates (or an iterator over all of them) can be reused in different +contexts to test using a number of invalid transaction types. + +Hopefully this makes it easier to get coverage of a full variety of tx +validation checks through different interfaces (AcceptBlock, AcceptToMemPool, +etc.) without repeating ourselves. + +Invalid tx cases not covered here can be found by running: + + $ diff \ + <(grep -IREho "bad-txns[a-zA-Z-]+" src | sort -u) \ + <(grep -IEho "bad-txns[a-zA-Z-]+" test/functional/data/invalid_txs.py | sort -u) + +""" +import abc + +from test_framework.messages import CTransaction, CTxIn, CTxOut, COutPoint +from test_framework import script as sc +from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS + +basic_p2sh = sc.CScript([sc.OP_HASH160, sc.hash160(sc.CScript([sc.OP_0])), sc.OP_EQUAL]) + + +class BadTxTemplate: + """Allows simple construction of a certain kind of invalid tx. Base class to be subclassed.""" + __metaclass__ = abc.ABCMeta + + # The expected error code given by bitcoind upon submission of the tx. + reject_reason = "" + + # Only specified if it differs from mempool acceptance error. + block_reject_reason = "" + + # Do we expect to be disconnected after submitting this tx? + expect_disconnect = False + + # Is this tx considered valid when included in a block, but not for acceptance into + # the mempool (i.e. does it violate policy but not consensus)? + valid_in_block = False + + def __init__(self, *, spend_tx=None, spend_block=None): + self.spend_tx = spend_block.vtx[0] if spend_block else spend_tx + self.spend_avail = sum(o.nValue for o in self.spend_tx.vout) + self.valid_txin = CTxIn(COutPoint(self.spend_tx.sha256, 0), b"", 0xffffffff) + + @abc.abstractmethod + def get_tx(self, *args, **kwargs): + """Return a CTransaction that is invalid per the subclass.""" + pass + + +class OutputMissing(BadTxTemplate): + reject_reason = "bad-txns-vout-empty" + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.calc_sha256() + return tx + + +class InputMissing(BadTxTemplate): + reject_reason = "bad-txns-vin-empty" + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE] * 100))) + tx.calc_sha256() + return tx + + +class SizeTooSmall(BadTxTemplate): + reject_reason = "tx-size-small" + expect_disconnect = False + valid_in_block = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE]))) + tx.calc_sha256() + return tx + + +class BadInputOutpointIndex(BadTxTemplate): + # Won't be rejected - nonexistent outpoint index is treated as an orphan since the coins + # database can't distinguish between spent outpoints and outpoints which never existed. + reject_reason = None + expect_disconnect = False + + def get_tx(self): + num_indices = len(self.spend_tx.vin) + bad_idx = num_indices + 100 + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256, bad_idx), b"", 0xffffffff)) + tx.vout.append(CTxOut(0, basic_p2sh)) + tx.calc_sha256() + return tx + + +class DuplicateInput(BadTxTemplate): + reject_reason = 'bad-txns-inputs-duplicate' + expect_disconnect = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + +class NonexistentInput(BadTxTemplate): + reject_reason = None # Added as an orphan tx. + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256 + 1, 0), b"", 0xffffffff)) + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + +class SpendTooMuch(BadTxTemplate): + reject_reason = 'bad-txns-in-belowout' + expect_disconnect = True + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_pub_key=basic_p2sh, amount=(self.spend_avail + 1)) + + +class SpendNegative(BadTxTemplate): + reject_reason = 'bad-txns-vout-negative' + expect_disconnect = True + + def get_tx(self): + return create_tx_with_script(self.spend_tx, 0, amount=-1) + + +class InvalidOPIFConstruction(BadTxTemplate): + reject_reason = "mandatory-script-verify-flag-failed (Invalid OP_IF construction)" + expect_disconnect = True + valid_in_block = True + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x64' * 35, + amount=(self.spend_avail // 2)) + + +class TooManySigops(BadTxTemplate): + reject_reason = "bad-txns-too-many-sigops" + block_reject_reason = "bad-blk-sigops, out-of-bounds SigOpCount" + expect_disconnect = False + + def get_tx(self): + lotsa_checksigs = sc.CScript([sc.OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) + return create_tx_with_script( + self.spend_tx, 0, + script_pub_key=lotsa_checksigs, + amount=1) + + +def iter_all_templates(): + """Iterate through all bad transaction template types.""" + return BadTxTemplate.__subclasses__() diff --git a/test/functional/data/rpc_psbt.json b/test/functional/data/rpc_psbt.json index dd40a096de..57f2608ee9 100644 --- a/test/functional/data/rpc_psbt.json +++ b/test/functional/data/rpc_psbt.json @@ -17,7 +17,8 @@ "cHNidP8BAJoCAAAAAljoeiG1ba8MI76OcHBFbDNvfLqlyHV5JPVFiHuyq911AAAAAAD/////g40EJ9DsZQpoqka7CwmK6kQiwHGyyng1Kgd5WdB86h0BAAAAAP////8CcKrwCAAAAAAWABTYXCtx0AYLCcmIauuBXlCZHdoSTQDh9QUAAAAAFgAUAK6pouXw+HaliN9VRuh0LR2HAI8AAAAAAAEAuwIAAAABqtc5MQGL0l+ErkALaISL4J23BurCrBgpi6vucatlb4sAAAAASEcwRAIgWPb8fGoz4bMVSNSByCbAFb0wE1qtQs1neQ2rZtKtJDsCIEoc7SYExnNbY5PltBaR3XiwDwxZQvufdRhW+qk4FX26Af7///8CgPD6AgAAAAAXqRQPuUY0IWlrgsgzryQceMF9295JNIfQ8gonAQAAABepFCnKdPigj4GZlCgYXJe12FLkBj9hh2UAAAABB9oARzBEAiB0AYrUGACXuHMyPAAVcgs2hMyBI4kQSOfbzZtVrWecmQIgc9Npt0Dj61Pc76M4I8gHBRTKVafdlUTxV8FnkTJhEYwBSDBFAiEA9hA4swjcHahlo0hSdG8BV3KTQgjG0kRUOTzZm98iF3cCIAVuZ1pnWm0KArhbFOXikHTYolqbV2C+ooFvZhkQoAbqAUdSIQKVg785rgpgl0etGZrd1jT6YQhVnWxc05tMIYPxq5bgfyEC2rYf9JoU22p9ArDNH7t4/EsYMStbTlTa5Nui+/71NtdSrgABASAAwusLAAAAABepFLf1+vQOPUClpFmx2zU18rcvqSHohwEHIyIAIIwjUxc3Q7WV37Sge3K6jkLjeX2nTof+fZ10l+OyAokDAQjaBABHMEQCIGLrelVhB6fHP0WsSrWh3d9vcHX7EnWWmn84Pv/3hLyyAiAMBdu3Rw2/LwhVfdNWxzJcHtMJE+mWzThAlF2xIijaXwFHMEQCIGX0W6WZi1mif/4ae+0BavHx+Q1Us6qPdFCqX1aiUQO9AiB/ckcDrR7blmgLKEtW1P/LiPf7dZ6rvgiqMPKbhROD0gFHUiEDCJ3BDHrG21T5EymvYXMz2ziM6tDCMfcjN50bmQMLAtwhAjrdkE89bc9Z3bkGsN7iNSm3/7ntUOXoYVGSaGAiHw5zUq4AIQIDqaTDf1mW06ol26xrVwrwZQOUSSlCRgs1R1PtnuylhxDZDGpPAAAAgAAAAIAEAACAACICAn9jmXV9Lv9VoTatAsaEsYOLZVbl8bazQoKpS2tQBRCWENkMak8AAACAAAAAgAUAAIAA", "cHNidP8BAHMCAAAAATAa6YblFqHsisW0vGVz0y+DtGXiOtdhZ9aLOOcwtNvbAAAAAAD/////AnR7AQAAAAAAF6kUA6oXrogrXQ1Usl1jEE5P/s57nqKHYEOZOwAAAAAXqRS5IbG6b3IuS/qDtlV6MTmYakLsg4cAAAAAAAEBHwDKmjsAAAAAFgAU0tlLZK4IWH7vyO6xh8YB6Tn5A3wCAwABAAAAAAEAFgAUYunpgv/zTdgjlhAxawkM0qO3R8sAAQAiACCHa62DLx0WgBXtQSMqnqZaGBXZ7xPA74dZ9ktbKyeKZQEBJVEhA7fOI6AcW0vwCmQlN836uzFbZoMyhnR471EwnSvVf4qHUa4A", "cHNidP8BAHMCAAAAATAa6YblFqHsisW0vGVz0y+DtGXiOtdhZ9aLOOcwtNvbAAAAAAD/////AnR7AQAAAAAAF6kUA6oXrogrXQ1Usl1jEE5P/s57nqKHYEOZOwAAAAAXqRS5IbG6b3IuS/qDtlV6MTmYakLsg4cAAAAAAAEBHwDKmjsAAAAAFgAU0tlLZK4IWH7vyO6xh8YB6Tn5A3wAAgAAFgAUYunpgv/zTdgjlhAxawkM0qO3R8sAAQAiACCHa62DLx0WgBXtQSMqnqZaGBXZ7xPA74dZ9ktbKyeKZQEBJVEhA7fOI6AcW0vwCmQlN836uzFbZoMyhnR471EwnSvVf4qHUa4A", - "cHNidP8BAHMCAAAAATAa6YblFqHsisW0vGVz0y+DtGXiOtdhZ9aLOOcwtNvbAAAAAAD/////AnR7AQAAAAAAF6kUA6oXrogrXQ1Usl1jEE5P/s57nqKHYEOZOwAAAAAXqRS5IbG6b3IuS/qDtlV6MTmYakLsg4cAAAAAAAEBHwDKmjsAAAAAFgAU0tlLZK4IWH7vyO6xh8YB6Tn5A3wAAQAWABRi6emC//NN2COWEDFrCQzSo7dHywABACIAIIdrrYMvHRaAFe1BIyqeploYFdnvE8Dvh1n2S1srJ4plIQEAJVEhA7fOI6AcW0vwCmQlN836uzFbZoMyhnR471EwnSvVf4qHUa4A" + "cHNidP8BAHMCAAAAATAa6YblFqHsisW0vGVz0y+DtGXiOtdhZ9aLOOcwtNvbAAAAAAD/////AnR7AQAAAAAAF6kUA6oXrogrXQ1Usl1jEE5P/s57nqKHYEOZOwAAAAAXqRS5IbG6b3IuS/qDtlV6MTmYakLsg4cAAAAAAAEBHwDKmjsAAAAAFgAU0tlLZK4IWH7vyO6xh8YB6Tn5A3wAAQAWABRi6emC//NN2COWEDFrCQzSo7dHywABACIAIIdrrYMvHRaAFe1BIyqeploYFdnvE8Dvh1n2S1srJ4plIQEAJVEhA7fOI6AcW0vwCmQlN836uzFbZoMyhnR471EwnSvVf4qHUa4A", + "cHNidP8BAHMCAAAAAbiWoY6pOQepFsEGhUPXaulX9rvye2NH+NrdlAHg+WgpAQAAAAD/////AkBLTAAAAAAAF6kUqWwXCcLM5BN2zoNqMNT5qMlIi7+HQEtMAAAAAAAXqRSVF/in2XNxAlN1OSxkyp0z+Wtg2YcAAAAAAAEBIBNssgAAAAAAF6kUamsvautR8hRlMRY6OKNTx03DK96HAQcXFgAUo8u1LWpHprjt/uENAwBpGZD0UH0BCGsCRzBEAiAONfH3DYiw67ZbylrsxCF/XXpVwyWBRgofyRbPslzvwgIgIKCsWw5sHSIPh1icNvcVLZLHWj6NA7Dk+4Os2pOnMbQBIQPGStfYHPtyhpV7zIWtn0Q4GXv5gK1zy/tnJ+cBXu4iiwABABYAFMwmJQEz+HDpBEEabxJ5PogPsqZRAAEAFgAUyCrGc3h3FYCmiIspbv2pSTKZ5jU" ], "valid" : [ "cHNidP8BAHUCAAAAASaBcTce3/KF6Tet7qSze3gADAVmy7OtZGQXE8pCFxv2AAAAAAD+////AtPf9QUAAAAAGXapFNDFmQPFusKGh2DpD9UhpGZap2UgiKwA4fUFAAAAABepFDVF5uM7gyxHBQ8k0+65PJwDlIvHh7MuEwAAAQD9pQEBAAAAAAECiaPHHqtNIOA3G7ukzGmPopXJRjr6Ljl/hTPMti+VZ+UBAAAAFxYAFL4Y0VKpsBIDna89p95PUzSe7LmF/////4b4qkOnHf8USIk6UwpyN+9rRgi7st0tAXHmOuxqSJC0AQAAABcWABT+Pp7xp0XpdNkCxDVZQ6vLNL1TU/////8CAMLrCwAAAAAZdqkUhc/xCX/Z4Ai7NK9wnGIZeziXikiIrHL++E4sAAAAF6kUM5cluiHv1irHU6m80GfWx6ajnQWHAkcwRAIgJxK+IuAnDzlPVoMR3HyppolwuAJf3TskAinwf4pfOiQCIAGLONfc0xTnNMkna9b7QPZzMlvEuqFEyADS8vAtsnZcASED0uFWdJQbrUqZY3LLh+GFbTZSYG2YVi/jnF6efkE/IQUCSDBFAiEA0SuFLYXc2WHS9fSrZgZU327tzHlMDDPOXMMJ/7X85Y0CIGczio4OFyXBl/saiK9Z9R5E5CVbIBZ8hoQDHAXR8lkqASECI7cr7vCWXRC+B3jv7NYfysb3mk6haTkzgHNEZPhPKrMAAAAAAAAA", diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 3f15367a75..f367e4fca8 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -13,7 +13,7 @@ is testing and *how* it's being tested # libraries then local imports). from collections import defaultdict -# Avoid wildcard * imports if possible +# Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv from test_framework.mininode import ( @@ -164,13 +164,13 @@ class ExampleTest(BitcoinTestFramework): self.tip = int(self.nodes[0].getbestblockhash(), 16) self.block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 - height = 1 + height = self.nodes[0].getblockcount() for i in range(10): # Use the mininode and blocktools functionality to manually build a block # Calling the generate() rpc is easier, but this allows us to exactly # control the blocks and transactions. - block = create_block(self.tip, create_coinbase(height), self.block_time) + block = create_block(self.tip, create_coinbase(height+1), self.block_time) block.solve() block_message = msg_block(block) # Send message is used to send a P2P message to the node over our P2PInterface diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 3d0467038d..12a4ce9aff 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -180,7 +180,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(120) + p2p1.sync_with_ping(150) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 628cefb76d..5253ff7aaa 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" @@ -7,7 +7,13 @@ import copy import struct import time -from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script, get_legacy_sigopcount_block +from test_framework.blocktools import ( + create_block, + create_coinbase, + create_tx_with_script, + get_legacy_sigopcount_block, + MAX_BLOCK_SIGOPS, +) from test_framework.key import CECKey from test_framework.messages import ( CBlock, @@ -45,8 +51,7 @@ from test_framework.script import ( ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal - -MAX_BLOCK_SIGOPS = 20000 +from data import invalid_txs # Use this class for tests that require behavior other than normal "mininode" behavior. # For now, it is used to serialize a bloated varint (b64). @@ -95,16 +100,21 @@ class FullBlockTest(BitcoinTestFramework): self.save_spendable_output() self.sync_blocks([b0]) + # These constants chosen specifically to trigger an immature coinbase spend + # at a certain time below. + NUM_BUFFER_BLOCKS_TO_GENERATE = 99 + NUM_OUTPUTS_TO_COLLECT = 33 + # Allow the block to mature blocks = [] - for i in range(99): - blocks.append(self.next_block(5000 + i)) + for i in range(NUM_BUFFER_BLOCKS_TO_GENERATE): + blocks.append(self.next_block("maturitybuffer.{}".format(i))) self.save_spendable_output() self.sync_blocks(blocks) # collect spendable outputs now to avoid cluttering the code later on out = [] - for i in range(33): + for i in range(NUM_OUTPUTS_TO_COLLECT): out.append(self.get_spendable_output()) # Start by building a couple of blocks on top (which output is spent is @@ -116,7 +126,48 @@ class FullBlockTest(BitcoinTestFramework): b2 = self.next_block(2, spend=out[1]) self.save_spendable_output() - self.sync_blocks([b1, b2]) + self.sync_blocks([b1, b2], timeout=4) + + # Select a txn with an output eligible for spending. This won't actually be spent, + # since we're testing submission of a series of blocks with invalid txns. + attempt_spend_tx = out[2] + + # Submit blocks for rejection, each of which contains a single transaction + # (aside from coinbase) which should be considered invalid. + for TxTemplate in invalid_txs.iter_all_templates(): + template = TxTemplate(spend_tx=attempt_spend_tx) + + if template.valid_in_block: + continue + + self.log.info("Reject block with invalid tx: %s", TxTemplate.__name__) + blockname = "for_invalid.%s" % TxTemplate.__name__ + badblock = self.next_block(blockname) + badtx = template.get_tx() + if TxTemplate != invalid_txs.InputMissing: + self.sign_tx(badtx, attempt_spend_tx) + else: + # Segwit is active in regtest at this point, so to deserialize a + # transaction without any inputs correctly, we set the outputs + # to an empty list. This is a hack, as the serialization of an + # empty list of outputs is deserialized as flags==0 and thus + # deserialization of the outputs is skipped. + # A policy check requires "loose" txs to be of a minimum size, + # so vtx is not set to be empty in the TxTemplate class and we + # only apply the workaround where txs are not "loose", i.e. in + # blocks. + # + # The workaround has the purpose that both sides calculate + # the same tx hash in the merkle tree + badtx.vout = [] + badtx.rehash() + badblock = self.update_block(blockname, [badtx]) + self.sync_blocks( + [badblock], success=False, + reject_reason=(template.block_reject_reason or template.reject_reason), + reconnect=True, timeout=2) + + self.move_tip(2) # Fork like this: # @@ -579,14 +630,14 @@ class FullBlockTest(BitcoinTestFramework): while b47.sha256 < target: b47.nNonce += 1 b47.rehash() - self.sync_blocks([b47], False, request_block=False) + self.sync_blocks([b47], False, force_send=True, reject_reason='high-hash') self.log.info("Reject a block with a timestamp >2 hours in the future") self.move_tip(44) b48 = self.next_block(48, solve=False) b48.nTime = int(time.time()) + 60 * 60 * 3 b48.solve() - self.sync_blocks([b48], False, request_block=False) + self.sync_blocks([b48], False, force_send=True, reject_reason='time-too-new') self.log.info("Reject a block with invalid merkle hash") self.move_tip(44) @@ -600,7 +651,7 @@ class FullBlockTest(BitcoinTestFramework): b50 = self.next_block(50) b50.nBits = b50.nBits - 1 b50.solve() - self.sync_blocks([b50], False, request_block=False, reconnect=True) + self.sync_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True) self.log.info("Reject a block with two coinbase transactions") self.move_tip(44) @@ -630,7 +681,7 @@ class FullBlockTest(BitcoinTestFramework): b54 = self.next_block(54, spend=out[15]) b54.nTime = b35.nTime - 1 b54.solve() - self.sync_blocks([b54], False, request_block=False) + self.sync_blocks([b54], False, force_send=True, reject_reason='time-too-old') # valid timestamp self.move_tip(53) @@ -824,7 +875,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) b64a = self.update_block("64a", [tx]) assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) - self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize():') + self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently # resend the header message, it won't send us the getdata message again. Just @@ -874,7 +925,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b67 (20) # # - self.log.info("Reject a block with a transaction double spending a transaction creted in the same block") + self.log.info("Reject a block with a transaction double spending a transaction created in the same block") self.move_tip(65) b67 = self.next_block(67) tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) @@ -1078,11 +1129,11 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(77) b80 = self.next_block(80, spend=out[25]) - self.sync_blocks([b80], False, request_block=False) + self.sync_blocks([b80], False, force_send=True) self.save_spendable_output() b81 = self.next_block(81, spend=out[26]) - self.sync_blocks([b81], False, request_block=False) # other chain is same length + self.sync_blocks([b81], False, force_send=True) # other chain is same length self.save_spendable_output() b82 = self.next_block(82, spend=out[27]) @@ -1169,7 +1220,7 @@ class FullBlockTest(BitcoinTestFramework): blocks = [] spend = out[32] for i in range(89, LARGE_REORG_SIZE + 89): - b = self.next_block(i, spend) + b = self.next_block(i, spend, version=4) tx = CTransaction() script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) @@ -1181,26 +1232,38 @@ class FullBlockTest(BitcoinTestFramework): self.save_spendable_output() spend = self.get_spendable_output() - self.sync_blocks(blocks, True, timeout=180) + self.sync_blocks(blocks, True, timeout=480) chain1_tip = i # now create alt chain of same length self.move_tip(88) blocks2 = [] for i in range(89, LARGE_REORG_SIZE + 89): - blocks2.append(self.next_block("alt" + str(i))) - self.sync_blocks(blocks2, False, request_block=False) + blocks2.append(self.next_block("alt" + str(i), version=4)) + self.sync_blocks(blocks2, False, force_send=True) # extend alt chain to trigger re-org - block = self.next_block("alt" + str(chain1_tip + 1)) - self.sync_blocks([block], True, timeout=180) + block = self.next_block("alt" + str(chain1_tip + 1), version=4) + self.sync_blocks([block], True, timeout=480) # ... and re-org back to the first chain self.move_tip(chain1_tip) - block = self.next_block(chain1_tip + 1) - self.sync_blocks([block], False, request_block=False) - block = self.next_block(chain1_tip + 2) - self.sync_blocks([block], True, timeout=180) + block = self.next_block(chain1_tip + 1, version=4) + self.sync_blocks([block], False, force_send=True) + block = self.next_block(chain1_tip + 2, version=4) + self.sync_blocks([block], True, timeout=480) + + self.log.info("Reject a block with an invalid block header version") + b_v1 = self.next_block('b_v1', version=1) + self.sync_blocks([b_v1], success=False, force_send=True, reject_reason='bad-version(0x00000001)') + + self.move_tip(chain1_tip + 2) + b_cb34 = self.next_block('b_cb34', version=4) + b_cb34.vtx[0].vin[0].scriptSig = b_cb34.vtx[0].vin[0].scriptSig[:-1] + b_cb34.vtx[0].rehash() + b_cb34.hashMerkleRoot = b_cb34.calc_merkle_root() + b_cb34.solve() + self.sync_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True) # Helper methods ################ @@ -1229,7 +1292,7 @@ class FullBlockTest(BitcoinTestFramework): tx.rehash() return tx - def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True): + def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True, *, version=1): if self.tip is None: base_block_hash = self.genesis_hash block_time = int(time.time()) + 1 @@ -1242,11 +1305,11 @@ class FullBlockTest(BitcoinTestFramework): coinbase.vout[0].nValue += additional_coinbase_value coinbase.rehash() if spend is None: - block = create_block(base_block_hash, coinbase, block_time) + block = create_block(base_block_hash, coinbase, block_time, version=version) else: coinbase.vout[0].nValue += spend.vout[0].nValue - 1 # all but one satoshi to fees coinbase.rehash() - block = create_block(base_block_hash, coinbase, block_time) + block = create_block(base_block_hash, coinbase, block_time, version=version) tx = self.create_tx(spend, 0, 1, script) # spend 1 satoshi self.sign_tx(tx, spend) self.add_transactions_to_block(block, [tx]) @@ -1288,7 +1351,7 @@ class FullBlockTest(BitcoinTestFramework): self.blocks[block_number] = block return block - def bootstrap_p2p(self): + def bootstrap_p2p(self, timeout=10): """Add a P2P connection to the node. Helper to connect and wait for version handshake.""" @@ -1299,24 +1362,25 @@ class FullBlockTest(BitcoinTestFramework): # an INV for the next block and receive two getheaders - one for the # IBD and one for the INV. We'd respond to both and could get # unexpectedly disconnected if the DoS score for that error is 50. - self.nodes[0].p2p.wait_for_getheaders(timeout=5) + self.nodes[0].p2p.wait_for_getheaders(timeout=timeout) - def reconnect_p2p(self): + def reconnect_p2p(self, timeout=60): """Tear down and bootstrap the P2P connection to the node. The node gets disconnected several times in this test. This helper method reconnects the p2p and restarts the network thread.""" self.nodes[0].disconnect_p2ps() - self.bootstrap_p2p() + self.bootstrap_p2p(timeout=timeout) - def sync_blocks(self, blocks, success=True, reject_reason=None, request_block=True, reconnect=False, timeout=60): + def sync_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" - self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_reason=reject_reason, request_block=request_block, timeout=timeout, expect_disconnect=reconnect) + self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_reason=reject_reason, force_send=force_send, timeout=timeout, expect_disconnect=reconnect) if reconnect: - self.reconnect_p2p() + self.reconnect_p2p(timeout=timeout) + if __name__ == '__main__': FullBlockTest().main() diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index c170f510c8..3a4889bbe9 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -18,6 +18,8 @@ class BlocksdirTest(BitcoinTestFramework): def run_test(self): self.stop_node(0) + assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest", "blocks")) + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "blocks")) shutil.rmtree(self.nodes[0].datadir) initialize_datadir(self.options.tmpdir, 0) self.log.info("Starting with nonexistent blocksdir ...") diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index f84b08a199..302a5ec1cb 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -25,7 +25,6 @@ CLTV_HEIGHT = 1351 # Reject codes that we might receive in this test REJECT_INVALID = 16 -REJECT_OBSOLETE = 17 REJECT_NONSTANDARD = 64 def cltv_invalidate(tx): diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 492772d5e3..4b3f6603a2 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -30,6 +30,23 @@ class ConfArgsTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: nono, if you intended to specify a negated option, use nono=1 instead') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('server=1\nrpcuser=someuser\nrpcpassword=some#pass') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided') + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('server=1\nrpcuser=someuser\nmain.rpcpassword=some#pass') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided') + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided') + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('testnot.datadir=1\n[testnet]\n') + self.restart_node(0) + self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.') + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear def run_test(self): diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index ae1eacf2d7..8b06cc7372 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -46,7 +46,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.num_nodes = 4 self.setup_clean_chain = False # Need a bit of extra time for the nodes to start up for this test - self.rpc_timewait = 90 + self.rpc_timeout = 90 # Set -maxmempool=0 to turn off mempool memory sharing with dbcache # Set -rpcservertimeout=900 to reduce socket disconnects in this @@ -69,6 +69,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): def setup_network(self): self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() + self.import_deterministic_coinbase_privkeys() # Leave them unconnected, we'll use submitblock directly in this test def restart_node(self, node_index, expected_tip): diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 16272877e7..9cbc1b39bd 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -22,7 +22,6 @@ DERSIG_HEIGHT = 1251 # Reject codes that we might receive in this test REJECT_INVALID = 16 -REJECT_OBSOLETE = 17 REJECT_NONSTANDARD = 64 # A canonical signature consists of: diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index aaab4279b5..b68e46adbc 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -144,6 +144,9 @@ class EstimateFeeTest(BitcoinTestFramework): # (68k weight is room enough for 120 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) + self.start_nodes() + self.import_deterministic_coinbase_privkeys() + self.stop_nodes() def transact_and_mine(self, numblocks, mining_node): min_fee = Decimal("0.00001") @@ -171,11 +174,6 @@ class EstimateFeeTest(BitcoinTestFramework): newmem.append(utx) self.memutxo = newmem - def import_deterministic_coinbase_privkeys(self): - self.start_nodes() - super().import_deterministic_coinbase_privkeys() - self.stop_nodes() - def run_test(self): self.log.info("This test is time consuming, please be patient") self.log.info("Splitting inputs so we can generate tx's") diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index a79cc3d34b..eb76089d9c 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -12,6 +12,7 @@ Generate 427 more blocks. [Consensus] Check that the new NULLDUMMY rules are not enforced on the 431st block. [Policy/Consensus] Check that the new NULLDUMMY rules are enforced on the 432nd block. """ +import time from test_framework.blocktools import create_coinbase, create_block, create_transaction, add_witness_commitment from test_framework.messages import CTransaction @@ -19,8 +20,6 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str -import time - NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" def trueDummy(tx): @@ -42,7 +41,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.setup_clean_chain = True # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through # normal segwit activation here (and don't use the default always-on behaviour). - self.extra_args = [['-whitelist=127.0.0.1', '-vbparams=segwit:0:999999999999', '-addresstype=legacy', "-deprecatedrpc=addwitnessaddress"]] + self.extra_args = [['-whitelist=127.0.0.1', '-vbparams=segwit:0:999999999999', '-addresstype=legacy']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -50,14 +49,14 @@ class NULLDUMMYTest(BitcoinTestFramework): def run_test(self): self.address = self.nodes[0].getnewaddress() self.ms_address = self.nodes[0].addmultisigaddress(1, [self.address])['address'] - self.wit_address = self.nodes[0].addwitnessaddress(self.address) + self.wit_address = self.nodes[0].getnewaddress(address_type='p2sh-segwit') self.wit_ms_address = self.nodes[0].addmultisigaddress(1, [self.address], '', 'p2sh-segwit')['address'] - self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 + self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 coinbase_txid = [] for i in self.coinbase_blocks: coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0]) - self.nodes[0].generate(427) # Block 429 + self.nodes[0].generate(427) # Block 429 self.lastblockhash = self.nodes[0].getbestblockhash() self.tip = int("0x" + self.lastblockhash, 0) self.lastblockheight = 429 @@ -82,7 +81,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) - test6txs=[CTransaction(test4tx)] + test6txs = [CTransaction(test4tx)] trueDummy(test4tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) self.block_submit(self.nodes[0], [test4tx]) @@ -99,8 +98,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) self.block_submit(self.nodes[0], test6txs, True, True) - - def block_submit(self, node, txs, witness = False, accept = False): + def block_submit(self, node, txs, witness=False, accept=False): block = create_block(self.tip, create_coinbase(self.lastblockheight + 1), self.lastblocktime + 1) block.nVersion = 4 for tx in txs: diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 772151dc4b..3e1ba88f0a 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -29,7 +29,7 @@ class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timewait = 900 + self.rpc_timeout = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. @@ -63,6 +63,8 @@ class PruneTest(BitcoinTestFramework): def setup_nodes(self): self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() + for n in self.nodes: + n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=False) def create_big_chain(self): # Start by creating some coinbases we can spend later @@ -189,6 +191,8 @@ class PruneTest(BitcoinTestFramework): def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) + with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(pruning, no data)']): + self.nodes[2].verifychain(checklevel=4, nblocks=0) self.log.info("Will need to redownload block %d" % self.forkheight) # Verify that we have enough history to reorg back to the fork point @@ -247,7 +251,7 @@ class PruneTest(BitcoinTestFramework): return index def prune(index, expected_ret=None): - ret = node.pruneblockchain(height(index)) + ret = node.pruneblockchain(height=height(index)) # Check the return value. When use_timestamp is True, just check # that the return value is less than or equal to the expected # value, because when more than one block is generated per second, @@ -316,7 +320,7 @@ class PruneTest(BitcoinTestFramework): if has_block(3): raise AssertionError("blk00003.dat is still there, should be pruned by now") - # stop node, start back up with auto-prune at 550MB, make sure still runs + # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) self.start_node(node_number, extra_args=["-prune=550"]) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 2cbfc26e89..1efc50e71f 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -5,11 +5,10 @@ """Test the SegWit changeover logic.""" from decimal import Decimal +from io import BytesIO from test_framework.address import ( key_to_p2pkh, - key_to_p2sh_p2wpkh, - key_to_p2wpkh, program_to_witness, script_to_p2sh, script_to_p2sh_p2wsh, @@ -21,8 +20,6 @@ from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash16 from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc -from io import BytesIO - NODE_0 = 0 NODE_2 = 2 WIT_V0 = 0 @@ -46,25 +43,26 @@ class SegWitTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [ [ "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-deprecatedrpc=addwitnessaddress", + "-txindex" ], [ "-blockversion=4", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-deprecatedrpc=addwitnessaddress", + "-txindex" ], [ "-blockversion=536870915", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-deprecatedrpc=addwitnessaddress", + "-txindex" ], ] @@ -91,40 +89,35 @@ class SegWitTest(BitcoinTestFramework): def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) - def run_test(self): - self.nodes[0].generate(161) #block 161 + self.nodes[0].generate(161) # block 161 self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tmpl = self.nodes[0].getblocktemplate({}) + tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) assert(tmpl['sizelimit'] == 1000000) assert('weightlimit' not in tmpl) assert(tmpl['sigoplimit'] == 20000) assert(tmpl['transactions'][0]['hash'] == txid) assert(tmpl['transactions'][0]['sigops'] == 2) - tmpl = self.nodes[0].getblocktemplate({'rules':['segwit']}) + tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) assert(tmpl['sizelimit'] == 1000000) assert('weightlimit' not in tmpl) assert(tmpl['sigoplimit'] == 20000) assert(tmpl['transactions'][0]['hash'] == txid) assert(tmpl['transactions'][0]['sigops'] == 2) - self.nodes[0].generate(1) #block 162 + self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() self.pubkey = [] - p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh - wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness + p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh + wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness for i in range(3): newaddress = self.nodes[i].getnewaddress() self.pubkey.append(self.nodes[i].getaddressinfo(newaddress)["pubkey"]) multiscript = CScript([OP_1, hex_str_to_bytes(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) - p2sh_addr = self.nodes[i].addwitnessaddress(newaddress) - bip173_addr = self.nodes[i].addwitnessaddress(newaddress, False) p2sh_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'p2sh-segwit')['address'] bip173_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'bech32')['address'] - assert_equal(p2sh_addr, key_to_p2sh_p2wpkh(self.pubkey[-1])) - assert_equal(bip173_addr, key_to_p2wpkh(self.pubkey[-1])) assert_equal(p2sh_ms_addr, script_to_p2sh_p2wsh(multiscript)) assert_equal(bip173_ms_addr, script_to_p2wsh(multiscript)) p2sh_ids.append([]) @@ -139,32 +132,32 @@ class SegWitTest(BitcoinTestFramework): wit_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], False, Decimal("49.999"))) p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) - self.nodes[0].generate(1) #block 163 + self.nodes[0].generate(1) # block 163 sync_blocks(self.nodes) # Make sure all nodes recognize the transactions as theirs - assert_equal(self.nodes[0].getbalance(), balance_presetup - 60*50 + 20*Decimal("49.999") + 50) - assert_equal(self.nodes[1].getbalance(), 20*Decimal("49.999")) - assert_equal(self.nodes[2].getbalance(), 20*Decimal("49.999")) + assert_equal(self.nodes[0].getbalance(), balance_presetup - 60 * 50 + 20 * Decimal("49.999") + 50) + assert_equal(self.nodes[1].getbalance(), 20 * Decimal("49.999")) + assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) - self.nodes[0].generate(260) #block 423 + self.nodes[0].generate(260) # block 423 sync_blocks(self.nodes) self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) #block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) #block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) #block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][0], True) #block 427 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) # block 424 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) # block 425 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) # block 426 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][0], True) # block 427 self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V0][1], False) self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V1][1], False) - self.nodes[2].generate(4) # blocks 428-431 + self.nodes[2].generate(4) # blocks 428-431 self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - block = self.nodes[2].generate(1) #block 432 (first block with new rules; 432 = 144 * 3) + block = self.nodes[2].generate(1) # block 432 (first block with new rules; 432 = 144 * 3) sync_blocks(self.nodes) assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(block[0])["tx"] @@ -181,8 +174,8 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) - assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) + assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) + assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) for i in range(len(segwit_tx_list)): tx = FromHex(CTransaction(), self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) assert(self.nodes[2].getrawtransaction(segwit_tx_list[i]) != self.nodes[0].getrawtransaction(segwit_tx_list[i])) @@ -198,21 +191,21 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness) (code 64)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) #block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) #block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) #block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) #block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) # block 432 + self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) # block 433 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) # block 434 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) # block 435 self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tmpl = self.nodes[0].getblocktemplate({'rules':['segwit']}) + tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data assert(tmpl['weightlimit'] == 4000000) assert(tmpl['sigoplimit'] == 80000) assert(tmpl['transactions'][0]['txid'] == txid) assert(tmpl['transactions'][0]['sigops'] == 8) - self.nodes[0].generate(1) # Mine a block to clear the gbt cache + self.nodes[0].generate(1) # Mine a block to clear the gbt cache self.log.info("Non-segwit miners are able to use GBT response after activation.") # Create a 3-tx chain: tx1 (non-segwit input, paying to a segwit output) -> @@ -222,7 +215,7 @@ class SegWitTest(BitcoinTestFramework): txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) hex_tx = self.nodes[0].gettransaction(txid)['hex'] tx = FromHex(CTransaction(), hex_tx) - assert(tx.wit.is_null()) # This should not be a segwit input + assert(tx.wit.is_null()) # This should not be a segwit input assert(txid1 in self.nodes[0].getrawmempool()) # Now create tx2, which will spend from txid1. @@ -243,17 +236,9 @@ class SegWitTest(BitcoinTestFramework): assert(tx.wit.is_null()) assert(txid3 in self.nodes[0].getrawmempool()) - # Now try calling getblocktemplate() without segwit support. - template = self.nodes[0].getblocktemplate() - - # Check that tx1 is the only transaction of the 3 in the template. - template_txids = [ t['txid'] for t in template['transactions'] ] - assert(txid2 not in template_txids and txid3 not in template_txids) - assert(txid1 in template_txids) - - # Check that running with segwit support results in all 3 being included. + # Check that getblocktemplate includes all transactions. template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) - template_txids = [ t['txid'] for t in template['transactions'] ] + template_txids = [t['txid'] for t in template['transactions']] assert(txid1 in template_txids) assert(txid2 in template_txids) assert(txid3 in template_txids) @@ -264,17 +249,17 @@ class SegWitTest(BitcoinTestFramework): # Mine a block to clear the gbt cache again. self.nodes[0].generate(1) - self.log.info("Verify behaviour of importaddress, addwitnessaddress and listunspent") + self.log.info("Verify behaviour of importaddress and listunspent") # Some public keys to be used later pubkeys = [ - "0363D44AABD0F1699138239DF2F042C3282C0671CC7A76826A55C8203D90E39242", # cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb - "02D3E626B3E616FC8662B489C123349FECBFC611E778E5BE739B257EAE4721E5BF", # cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97 - "04A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538A62F5BD8EC85C2477F39650BD391EA6250207065B2A81DA8B009FC891E898F0E", # 91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV - "02A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538", # cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd - "036722F784214129FEB9E8129D626324F3F6716555B603FFE8300BBCB882151228", # cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66 - "0266A8396EE936BF6D99D17920DB21C6C7B1AB14C639D5CD72B300297E416FD2EC", # cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K - "0450A38BD7F0AC212FEBA77354A9B036A32E0F7C81FC4E0C5ADCA7C549C4505D2522458C2D9AE3CEFD684E039194B72C8A10F9CB9D4764AB26FCC2718D421D3B84", # 92h2XPssjBpsJN5CqSP7v9a7cf2kgDunBC6PDFwJHMACM1rrVBJ + "0363D44AABD0F1699138239DF2F042C3282C0671CC7A76826A55C8203D90E39242", # cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb + "02D3E626B3E616FC8662B489C123349FECBFC611E778E5BE739B257EAE4721E5BF", # cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97 + "04A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538A62F5BD8EC85C2477F39650BD391EA6250207065B2A81DA8B009FC891E898F0E", # 91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV + "02A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538", # cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd + "036722F784214129FEB9E8129D626324F3F6716555B603FFE8300BBCB882151228", # cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66 + "0266A8396EE936BF6D99D17920DB21C6C7B1AB14C639D5CD72B300297E416FD2EC", # cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K + "0450A38BD7F0AC212FEBA77354A9B036A32E0F7C81FC4E0C5ADCA7C549C4505D2522458C2D9AE3CEFD684E039194B72C8A10F9CB9D4764AB26FCC2718D421D3B84", # 92h2XPssjBpsJN5CqSP7v9a7cf2kgDunBC6PDFwJHMACM1rrVBJ ] # Import a compressed key and an uncompressed key, generate some multisig addresses @@ -282,8 +267,8 @@ class SegWitTest(BitcoinTestFramework): uncompressed_spendable_address = ["mvozP4UwyGD2mGZU4D2eMvMLPB9WkMmMQu"] self.nodes[0].importprivkey("cNC8eQ5dg3mFAVePDX4ddmPYpPbw41r9bm2jd1nLJT77e6RrzTRR") compressed_spendable_address = ["mmWQubrDomqpgSYekvsU7HWEVjLFHAakLe"] - assert ((self.nodes[0].getaddressinfo(uncompressed_spendable_address[0])['iscompressed'] == False)) - assert ((self.nodes[0].getaddressinfo(compressed_spendable_address[0])['iscompressed'] == True)) + assert not self.nodes[0].getaddressinfo(uncompressed_spendable_address[0])['iscompressed'] + assert self.nodes[0].getaddressinfo(compressed_spendable_address[0])['iscompressed'] self.nodes[0].importpubkey(pubkeys[0]) compressed_solvable_address = [key_to_p2pkh(pubkeys[0])] @@ -305,7 +290,6 @@ class SegWitTest(BitcoinTestFramework): uncompressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], uncompressed_solvable_address[0]])['address']) compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_solvable_address[0]])['address']) compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_solvable_address[0], compressed_solvable_address[1]])['address']) - unknown_address = ["mtKKyoHabkk6e4ppT7NaM7THqPUt7AzPrT", "2NDP3jLWAFT8NDAiUa9qiE6oBt2awmMq7Dx"] # Test multisig_without_privkey # We have 2 public keys without private keys, use addmultisigaddress to add to wallet. @@ -386,7 +370,6 @@ class SegWitTest(BitcoinTestFramework): op1 = CScript([OP_1]) op0 = CScript([OP_0]) # 2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe is the P2SH(P2PKH) version of mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V - unsolvable_address = ["mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V", "2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe", script_to_p2sh(op1), script_to_p2sh(op0)] unsolvable_address_key = hex_str_to_bytes("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") unsolvablep2pkh = CScript([OP_DUP, OP_HASH160, hash160(unsolvable_address_key), OP_EQUALVERIFY, OP_CHECKSIG]) unsolvablep2wshp2pkh = CScript([OP_0, sha256(unsolvablep2pkh)]) @@ -394,9 +377,9 @@ class SegWitTest(BitcoinTestFramework): p2wshop1 = CScript([OP_0, sha256(op1)]) unsolvable_after_importaddress.append(unsolvablep2pkh) unsolvable_after_importaddress.append(unsolvablep2wshp2pkh) - unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script + unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script unsolvable_after_importaddress.append(p2wshop1) - unseen_anytime.append(op0) # OP_0 will be imported as P2SH address with no script provided + unseen_anytime.append(op0) # OP_0 will be imported as P2SH address with no script provided unsolvable_after_importaddress.append(p2shop0) spendable_txid = [] @@ -432,27 +415,14 @@ class SegWitTest(BitcoinTestFramework): # exceptions and continue. try_rpc(-4, "The wallet already contains the private key for this address or script", self.nodes[0].importaddress, i, "", False, True) - self.nodes[0].importaddress(script_to_p2sh(op0)) # import OP_0 as address only - self.nodes[0].importaddress(multisig_without_privkey_address) # Test multisig_without_privkey + self.nodes[0].importaddress(script_to_p2sh(op0)) # import OP_0 as address only + self.nodes[0].importaddress(multisig_without_privkey_address) # Test multisig_without_privkey spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) self.mine_and_test_listunspent(unseen_anytime, 0) - # addwitnessaddress should refuse to return a witness address if an uncompressed key is used - # note that no witness address should be returned by unsolvable addresses - for i in uncompressed_spendable_address + uncompressed_solvable_address + unknown_address + unsolvable_address: - assert_raises_rpc_error(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) - - # addwitnessaddress should return a witness addresses even if keys are not in the wallet - self.nodes[0].addwitnessaddress(multisig_without_privkey_address) - - for i in compressed_spendable_address + compressed_solvable_address: - witaddress = self.nodes[0].addwitnessaddress(i) - # addwitnessaddress should return the same address if it is a known P2SH-witness address - assert_equal(witaddress, self.nodes[0].addwitnessaddress(witaddress)) - spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) @@ -470,8 +440,6 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].importpubkey(pubkeys[6]) uncompressed_solvable_address = [key_to_p2pkh(pubkeys[6])] - spendable_after_addwitnessaddress = [] # These outputs should be seen after importaddress - solvable_after_addwitnessaddress=[] # These outputs should be seen after importaddress but not spendable unseen_anytime = [] # These outputs should never be seen solvable_anytime = [] # These outputs should be solvable after importpubkey unseen_anytime = [] # These outputs should never be seen @@ -488,8 +456,6 @@ class SegWitTest(BitcoinTestFramework): v = self.nodes[0].getaddressinfo(i) if (v['isscript']): [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - # P2WSH and P2SH(P2WSH) multisig with compressed keys are spendable after addwitnessaddress - spendable_after_addwitnessaddress.extend([p2wsh, p2sh_p2wsh]) premature_witaddress.append(script_to_p2sh(p2wsh)) else: [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) @@ -510,9 +476,7 @@ class SegWitTest(BitcoinTestFramework): for i in compressed_solvable_address: v = self.nodes[0].getaddressinfo(i) if (v['isscript']): - # P2WSH multisig without private key are seen after addwitnessaddress [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - solvable_after_addwitnessaddress.extend([p2wsh, p2sh_p2wsh]) premature_witaddress.append(script_to_p2sh(p2wsh)) else: [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) @@ -521,29 +485,11 @@ class SegWitTest(BitcoinTestFramework): self.mine_and_test_listunspent(spendable_anytime, 2) self.mine_and_test_listunspent(solvable_anytime, 1) - self.mine_and_test_listunspent(spendable_after_addwitnessaddress + solvable_after_addwitnessaddress + unseen_anytime, 0) - - # addwitnessaddress should refuse to return a witness address if an uncompressed key is used - # note that a multisig address returned by addmultisigaddress is not solvable until it is added with importaddress - # premature_witaddress are not accepted until the script is added with addwitnessaddress first - for i in uncompressed_spendable_address + uncompressed_solvable_address + premature_witaddress: - # This will raise an exception - assert_raises_rpc_error(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) - - # after importaddress it should pass addwitnessaddress - v = self.nodes[0].getaddressinfo(compressed_solvable_address[1]) - self.nodes[0].importaddress(v['hex'],"",False,True) - for i in compressed_spendable_address + compressed_solvable_address + premature_witaddress: - witaddress = self.nodes[0].addwitnessaddress(i) - assert_equal(witaddress, self.nodes[0].addwitnessaddress(witaddress)) - - spendable_txid.append(self.mine_and_test_listunspent(spendable_after_addwitnessaddress + spendable_anytime, 2)) - solvable_txid.append(self.mine_and_test_listunspent(solvable_after_addwitnessaddress + solvable_anytime, 1)) self.mine_and_test_listunspent(unseen_anytime, 0) # Check that createrawtransaction/decoderawtransaction with non-v0 Bech32 works - v1_addr = program_to_witness(1, [3,5]) - v1_tx = self.nodes[0].createrawtransaction([getutxo(spendable_txid[0])],{v1_addr: 1}) + v1_addr = program_to_witness(1, [3, 5]) + v1_tx = self.nodes[0].createrawtransaction([getutxo(spendable_txid[0])], {v1_addr: 1}) v1_decoded = self.nodes[1].decoderawtransaction(v1_tx) assert_equal(v1_decoded['vout'][0]['scriptPubKey']['addresses'][0], v1_addr) assert_equal(v1_decoded['vout'][0]['scriptPubKey']['hex'], "51020305") @@ -586,7 +532,7 @@ class SegWitTest(BitcoinTestFramework): def mine_and_test_listunspent(self, script_list, ismine): utxo = find_spendable_utxo(self.nodes[0], 50) tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(int('0x'+utxo['txid'],0), utxo['vout']))) + tx.vin.append(CTxIn(COutPoint(int('0x' + utxo['txid'], 0), utxo['vout']))) for i in script_list: tx.vout.append(CTxOut(10000000, i)) tx.rehash() @@ -599,7 +545,7 @@ class SegWitTest(BitcoinTestFramework): for i in self.nodes[0].listunspent(): if (i['txid'] == txid): watchcount += 1 - if (i['spendable'] == True): + if i['spendable']: spendcount += 1 if (ismine == 2): assert_equal(spendcount, len(script_list)) @@ -610,14 +556,14 @@ class SegWitTest(BitcoinTestFramework): assert_equal(watchcount, 0) return txid - def p2sh_address_to_script(self,v): + def p2sh_address_to_script(self, v): bare = CScript(hex_str_to_bytes(v['hex'])) p2sh = CScript(hex_str_to_bytes(v['scriptPubKey'])) p2wsh = CScript([OP_0, sha256(bare)]) p2sh_p2wsh = CScript([OP_HASH160, hash160(p2wsh), OP_EQUAL]) return([bare, p2sh, p2wsh, p2sh_p2wsh]) - def p2pkh_address_to_script(self,v): + def p2pkh_address_to_script(self, v): pubkey = hex_str_to_bytes(v['pubkey']) p2wpkh = CScript([OP_0, hash160(pubkey)]) p2sh_p2wpkh = CScript([OP_HASH160, hash160(p2wpkh), OP_EQUAL]) @@ -631,7 +577,7 @@ class SegWitTest(BitcoinTestFramework): p2sh_p2wsh_p2pkh = CScript([OP_HASH160, hash160(p2wsh_p2pkh), OP_EQUAL]) return [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] - def create_and_mine_tx_from_txids(self, txids, success = True): + def create_and_mine_tx_from_txids(self, txids, success=True): tx = CTransaction() for i in txids: txtmp = CTransaction() @@ -639,7 +585,7 @@ class SegWitTest(BitcoinTestFramework): f = BytesIO(hex_str_to_bytes(txraw)) txtmp.deserialize(f) for j in range(len(txtmp.vout)): - tx.vin.append(CTxIn(COutPoint(int('0x'+i,0), j))) + tx.vin.append(CTxIn(COutPoint(int('0x' + i, 0), j))) tx.vout.append(CTxOut(0, CScript())) tx.rehash() signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py new file mode 100755 index 0000000000..5084cb1322 --- /dev/null +++ b/test/functional/feature_shutdown.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoind shutdown.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, get_rpc_proxy, wait_until +from threading import Thread + +def test_long_call(node): + block = node.waitfornewblock() + assert_equal(block['height'], 0) + +class ShutdownTest(BitcoinTestFramework): + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + node = get_rpc_proxy(self.nodes[0].url, 1, timeout=600, coveragedir=self.nodes[0].coverage_dir) + # Force connection establishment by executing a dummy command. + node.getblockcount() + Thread(target=test_long_call, args=(node,)).start() + # Wait until the server is executing the above `waitfornewblock`. + wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) + # Wait 1 second after requesting shutdown but not before the `stop` call + # finishes. This is to ensure event loop waits for current connections + # to close. + self.stop_node(0, wait=1000) + +if __name__ == '__main__': + ShutdownTest().main() diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index e4b86f9e1e..20889366e5 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -31,13 +31,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -48,13 +48,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -65,7 +65,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock==None) #now the connection must be closed after the response + assert(conn.sock is None) #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -88,7 +88,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #connection must be closed because bitcoind should use keep-alive by default + assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index afa9de580f..f850a54462 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -41,7 +41,8 @@ class RESTTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-rest"], []] + # TODO: remove -txindex. Currently required for getrawtransaction call. + self.extra_args = [["-rest", "-txindex"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -198,9 +199,19 @@ class RESTTest (BitcoinTestFramework): self.nodes[0].generate(1) # generate block to not affect upcoming tests self.sync_all() - self.log.info("Test the /block and /headers URIs") + self.log.info("Test the /block, /blockhashbyheight and /headers URIs") bb_hash = self.nodes[0].getbestblockhash() + # Check result if block does not exists + assert_equal(self.test_rest_request('/headers/1/0000000000000000000000000000000000000000000000000000000000000000'), []) + self.test_rest_request('/block/0000000000000000000000000000000000000000000000000000000000000000', status=404, ret_type=RetType.OBJ) + + # Check result if block is not in the active chain + self.nodes[0].invalidateblock(bb_hash) + assert_equal(self.test_rest_request('/headers/1/{}'.format(bb_hash)), []) + self.test_rest_request('/block/{}'.format(bb_hash)) + self.nodes[0].reconsiderblock(bb_hash) + # Check binary format response = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) assert_greater_than(int(response.getheader('content-length')), 80) @@ -227,6 +238,23 @@ class RESTTest (BitcoinTestFramework): # Check json format block_json_obj = self.test_rest_request("/block/{}".format(bb_hash)) assert_equal(block_json_obj['hash'], bb_hash) + assert_equal(self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']))['blockhash'], bb_hash) + + # Check hex/bin format + resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) + assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) + resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) + blockhash = binascii.hexlify(resp_bytes[::-1]).decode('utf-8') + assert_equal(blockhash, bb_hash) + + # Check invalid blockhashbyheight requests + resp = self.test_rest_request("/blockhashbyheight/abc", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: abc") + resp = self.test_rest_request("/blockhashbyheight/1000000", ret_type=RetType.OBJ, status=404) + assert_equal(resp.read().decode('utf-8').rstrip(), "Block height out of range") + resp = self.test_rest_request("/blockhashbyheight/-1", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: -1") + self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header json_obj = self.test_rest_request("/headers/1/{}".format(bb_hash)) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py new file mode 100755 index 0000000000..b6955d4492 --- /dev/null +++ b/test/functional/interface_rpc.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests some generic aspects of the RPC interface.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_greater_than_or_equal + +class RPCInterfaceTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def test_getrpcinfo(self): + self.log.info("Testing getrpcinfo...") + + info = self.nodes[0].getrpcinfo() + assert_equal(len(info['active_commands']), 1) + + command = info['active_commands'][0] + assert_equal(command['method'], 'getrpcinfo') + assert_greater_than_or_equal(command['duration'], 0) + + def test_batch_request(self): + self.log.info("Testing basic JSON-RPC batch request...") + + results = self.nodes[0].batch([ + # A basic request that will work fine. + {"method": "getblockcount", "id": 1}, + # Request that will fail. The whole batch request should still + # work fine. + {"method": "invalidmethod", "id": 2}, + # Another call that should succeed. + {"method": "getbestblockhash", "id": 3}, + ]) + + result_by_id = {} + for res in results: + result_by_id[res["id"]] = res + + assert_equal(result_by_id[1]['error'], None) + assert_equal(result_by_id[1]['result'], 0) + + assert_equal(result_by_id[2]['error']['code'], -32601) + assert_equal(result_by_id[2]['result'], None) + + assert_equal(result_by_id[3]['error'], None) + assert result_by_id[3]['result'] is not None + + def run_test(self): + self.test_getrpcinfo() + self.test_batch_request() + + +if __name__ == '__main__': + RPCInterfaceTest().main() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 48136a0108..94fea37090 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -69,6 +69,7 @@ class ZMQTest (BitcoinTestFramework): ] self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() + self.import_deterministic_coinbase_privkeys() def run_test(self): try: @@ -121,10 +122,10 @@ class ZMQTest (BitcoinTestFramework): self.log.info("Test the getzmqnotifications RPC") assert_equal(self.nodes[0].getzmqnotifications(), [ - {"type": "pubhashblock", "address": ADDRESS}, - {"type": "pubhashtx", "address": ADDRESS}, - {"type": "pubrawblock", "address": ADDRESS}, - {"type": "pubrawtx", "address": ADDRESS}, + {"type": "pubhashblock", "address": ADDRESS, "hwm": 1000}, + {"type": "pubhashtx", "address": ADDRESS, "hwm": 1000}, + {"type": "pubrawblock", "address": ADDRESS, "hwm": 1000}, + {"type": "pubrawtx", "address": ADDRESS, "hwm": 1000}, ]) assert_equal(self.nodes[1].getzmqnotifications(), []) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 8847777ba7..e2a219b85a 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" from io import BytesIO +import math + from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, @@ -56,6 +58,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.mempool_size = 0 wait_until(lambda: node.getblockcount() == 200) assert_equal(node.getmempoolinfo()['size'], self.mempool_size) + coins = node.listunspent() self.log.info('Should not accept garbage to testmempoolaccept') assert_raises_rpc_error(-3, 'Expected type array, got string', lambda: node.testmempoolaccept(rawtxs='ff00baar')) @@ -63,13 +66,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): assert_raises_rpc_error(-22, 'TX decode failed', lambda: node.testmempoolaccept(rawtxs=['ff00baar'])) self.log.info('A transaction already in the blockchain') - coin = node.listunspent()[0] # Pick a random coin(base) to spend + coin = coins.pop() # Pick a random coin(base) to spend raw_tx_in_block = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[{'txid': coin['txid'], 'vout': coin['vout']}], outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], ))['hex'] txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True) node.generate(1) + self.mempool_size = 0 self.check_mempool_result( result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': '18: txn-already-known'}], rawtxs=[raw_tx_in_block], @@ -89,9 +93,25 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[raw_tx_0], ) + self.log.info('A final transaction not in the mempool') + coin = coins.pop() # Pick a random coin(base) to spend + raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction( + inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}], # SEQUENCE_FINAL + outputs=[{node.getnewaddress(): 0.025}], + locktime=node.getblockcount() + 2000, # Can be anything + ))['hex'] + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': True}], + rawtxs=[bytes_to_hex_str(tx.serialize())], + allowhighfees=True, + ) + node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True) + self.mempool_size += 1 + self.log.info('A transaction in the mempool') node.sendrawtransaction(hexstring=raw_tx_0) - self.mempool_size = 1 + self.mempool_size += 1 self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': '18: txn-already-in-mempool'}], rawtxs=[raw_tx_0], @@ -181,7 +201,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A really large transaction') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) - tx.vin = [tx.vin[0]] * (MAX_BLOCK_BASE_SIZE // len(tx.vin[0].serialize())) + tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], rawtxs=[bytes_to_hex_str(tx.serialize())], diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index b4e9d967fd..d74d4eaaf1 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -42,6 +42,7 @@ import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, wait_until + class MempoolPersistTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 @@ -60,7 +61,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Send 5 transactions from node2 (to its own address)") for i in range(5): - self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) node2_balance = self.nodes[2].getbalance() self.sync_all() @@ -68,6 +69,13 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + self.log.debug("Prioritize a transaction on node0") + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'], fees['modified']) + self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000) + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -81,6 +89,10 @@ class MempoolPersistTest(BitcoinTestFramework): # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) + self.log.debug('Verify prioritization is loaded correctly') + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + # Verify accounting of mempool transactions after restart is correct self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index d035ca907a..845beb551e 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -47,12 +47,11 @@ class MempoolCoinbaseTest(BitcoinTestFramework): tx = self.nodes[0].gettransaction(txid) assert(tx["confirmations"] > 0) - # Use invalidateblock to re-org back; all transactions should - # end up unconfirmed and back in the mempool + # Use invalidateblock to re-org back for node in self.nodes: node.invalidateblock(blocks[0]) - # mempool should be empty, all txns confirmed + # All txns should be back in mempool with 0 confirmations assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index ff55ea5528..661d9f4c97 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -25,14 +25,15 @@ from test_framework.util import ( assert_raises_rpc_error, bytes_to_hex_str as b2x, ) - +from test_framework.script import CScriptNum def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() - rsp = node.getblocktemplate({'data': b2x(block.serialize()), 'mode': 'proposal'}) + rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']}) assert_equal(rsp, expect) + class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -59,16 +60,24 @@ class MiningTest(BitcoinTestFramework): # Mine a block to leave initial block download node.generatetoaddress(1, node.get_deterministic_priv_key().address) - tmpl = node.getblocktemplate() + tmpl = node.getblocktemplate({'rules': ['segwit']}) self.log.info("getblocktemplate: Test capability advertised") assert 'proposal' in tmpl['capabilities'] assert 'coinbasetxn' not in tmpl - coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1) + next_height = int(tmpl["height"]) + coinbase_tx = create_coinbase(height=next_height) # sequence numbers must not be max for nLockTime to have effect coinbase_tx.vin[0].nSequence = 2 ** 32 - 2 coinbase_tx.rehash() + # round-trip the encoded bip34 block height commitment + assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height) + # round-trip negative and multi-byte CScriptNums to catch python regression + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500) + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500) + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1) + block = CBlock() block.nVersion = tmpl["version"] block.hashPrevBlock = int(tmpl["previousblockhash"], 16) @@ -77,6 +86,9 @@ class MiningTest(BitcoinTestFramework): block.nNonce = 0 block.vtx = [coinbase_tx] + self.log.info("getblocktemplate: segwit rule must be set") + assert_raises_rpc_error(-8, "getblocktemplate must be called with the segwit rule set", node.getblocktemplate) + self.log.info("getblocktemplate: Test valid block") assert_template(node, block, None) @@ -93,7 +105,7 @@ class MiningTest(BitcoinTestFramework): assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -123,7 +135,7 @@ class MiningTest(BitcoinTestFramework): bad_block_sn = bytearray(block.serialize()) assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) bad_block_sn[TX_COUNT_OFFSET] += 1 - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 9a3c15a4a7..72cde8e811 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -15,14 +15,14 @@ class LongpollThread(threading.Thread): def __init__(self, node): threading.Thread.__init__(self) # query current longpollid - template = node.getblocktemplate() + template = node.getblocktemplate({'rules': ['segwit']}) self.longpollid = template['longpollid'] # create a new connection to the node, we can't use the same # connection from two threads self.node = get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) def run(self): - self.node.getblocktemplate({'longpollid':self.longpollid}) + self.node.getblocktemplate({'longpollid': self.longpollid, 'rules': ['segwit']}) class GetBlockTemplateLPTest(BitcoinTestFramework): def set_test_params(self): @@ -34,10 +34,10 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): def run_test(self): self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.") self.nodes[0].generate(10) - template = self.nodes[0].getblocktemplate() + template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) longpollid = template['longpollid'] # longpollid should not change between successive invocations if nothing else happens - template2 = self.nodes[0].getblocktemplate() + template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) assert(template2['longpollid'] == longpollid) # Test 1: test that the longpolling wait if we do nothing diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index c5ddee56f1..ca4b621a78 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx != None) + assert(high_fee_tx is not None) # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). @@ -142,10 +142,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # getblocktemplate to (eventually) return a new block. mock_time = int(time.time()) self.nodes[0].setmocktime(mock_time) - template = self.nodes[0].getblocktemplate() + template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN)) self.nodes[0].setmocktime(mock_time+10) - new_template = self.nodes[0].getblocktemplate() + new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) assert(template != new_template) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index fafc27d16a..f0dee59b5c 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -7,7 +7,6 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ - from decimal import Decimal import random @@ -99,7 +98,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.num_nodes = 2 # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex", "-deprecatedrpc=addwitnessaddress"]] + self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] self.utxos = [] def skip_test_if_missing_module(self): @@ -189,7 +188,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now try a SENDCMPCT message with too-high version sendcmpct = msg_sendcmpct() - sendcmpct.version = preferred_version+1 + sendcmpct.version = preferred_version + 1 sendcmpct.announce = True test_node.send_and_ping(sendcmpct) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) @@ -220,7 +219,7 @@ class CompactBlocksTest(BitcoinTestFramework): check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time, after sending a version-1, announce=false message. - sendcmpct.version = preferred_version-1 + sendcmpct.version = preferred_version - 1 sendcmpct.announce = False test_node.send_and_ping(sendcmpct) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) @@ -234,7 +233,7 @@ class CompactBlocksTest(BitcoinTestFramework): if old_node is not None: # Verify that a peer using an older protocol version can receive # announcements from this node. - sendcmpct.version = preferred_version-1 + sendcmpct.version = preferred_version - 1 sendcmpct.announce = True old_node.send_and_ping(sendcmpct) # Header sync @@ -265,9 +264,9 @@ class CompactBlocksTest(BitcoinTestFramework): if use_witness_address: # Want at least one segwit spend, so move all funds to # a witness address. - address = node.addwitnessaddress(address) + address = node.getnewaddress(address_type='bech32') value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1))) + node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) node.generate(1) segwit_tx_generated = False @@ -279,7 +278,7 @@ class CompactBlocksTest(BitcoinTestFramework): segwit_tx_generated = True if use_witness_address: - assert(segwit_tx_generated) # check that our test is not broken + assert segwit_tx_generated # check that our test is not broken # Wait until we've seen the block announcement for the resulting tip tip = int(node.getbestblockhash(), 16) @@ -403,8 +402,7 @@ class CompactBlocksTest(BitcoinTestFramework): coinbase_hash = block.vtx[0].sha256 if version == 2: coinbase_hash = block.vtx[0].calc_sha256(True) - comp_block.shortids = [ - calculate_shortid(k0, k1, coinbase_hash) ] + comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. @@ -443,7 +441,7 @@ class CompactBlocksTest(BitcoinTestFramework): # node needs, and that responding to them causes the block to be # reconstructed. def test_getblocktxn_requests(self, node, test_node, version): - with_witness = (version==2) + with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) @@ -470,7 +468,7 @@ class CompactBlocksTest(BitcoinTestFramework): msg_bt = msg_blocktxn() if with_witness: - msg_bt = msg_witness_blocktxn() # serialize with witnesses + msg_bt = msg_witness_blocktxn() # serialize with witnesses msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) test_tip_after_message(node, test_node, msg_bt, block.sha256) @@ -560,7 +558,7 @@ class CompactBlocksTest(BitcoinTestFramework): # verifying that the block isn't marked bad permanently. This is good # enough for now. msg = msg_blocktxn() - if version==2: + if version == 2: msg = msg_witness_blocktxn() msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) test_node.send_and_ping(msg) @@ -571,11 +569,11 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG) + assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG) assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block - if version==2: + if version == 2: test_node.send_and_ping(msg_witness_block(block)) else: test_node.send_and_ping(msg_block(block)) @@ -655,7 +653,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Generate an old compactblock, and verify that it's not accepted. cur_height = node.getblockcount() - hashPrevBlock = int(node.getblockhash(cur_height-5), 16) + hashPrevBlock = int(node.getblockhash(cur_height - 5), 16) block = self.build_block_on_tip(node) block.hashPrevBlock = hashPrevBlock block.solve() @@ -684,7 +682,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert "blocktxn" not in test_node.last_message def activate_segwit(self, node): - node.generate(144*3) + node.generate(144 * 3) assert_equal(get_bip9_status(node, "segwit")["status"], 'active') def test_end_to_end_block_relay(self, node, listeners): @@ -780,7 +778,7 @@ class CompactBlocksTest(BitcoinTestFramework): delivery_peer.send_message(msg_tx(tx)) delivery_peer.sync_with_ping() - cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ] + cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()] cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] cmpct_block.use_witness = True @@ -886,7 +884,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Syncing nodes...") assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1) + block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 67f24d6bff..1b11a2a294 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -22,7 +22,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: successfully ban single IP address") assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point - self.nodes[1].setban("127.0.0.1", "add") + self.nodes[1].setban(subnet="127.0.0.1", command="add") wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].listbanned()), 1) diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 0678b1a651..1e0b876593 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -77,9 +77,9 @@ class InvalidBlockRequestTest(BitcoinTestFramework): block2.vtx.append(tx2) assert_equal(block2.hashMerkleRoot, block2.calc_merkle_root()) assert_equal(orig_hash, block2.rehash()) - assert(block2_orig.vtx != block2.vtx) + assert block2_orig.vtx != block2.vtx - node.p2p.send_blocks_and_test([block2], node, success=False, request_block=False, reject_reason='bad-txns-duplicate') + node.p2p.send_blocks_and_test([block2], node, success=False, reject_reason='bad-txns-duplicate') # Check transactions for duplicate inputs self.log.info("Test duplicate input block.") @@ -89,7 +89,7 @@ class InvalidBlockRequestTest(BitcoinTestFramework): block2_orig.hashMerkleRoot = block2_orig.calc_merkle_root() block2_orig.rehash() block2_orig.solve() - node.p2p.send_blocks_and_test([block2_orig], node, success=False, request_block=False, reject_reason='bad-txns-inputs-duplicate') + node.p2p.send_blocks_and_test([block2_orig], node, success=False, reject_reason='bad-txns-inputs-duplicate') self.log.info("Test very broken block.") @@ -102,7 +102,8 @@ class InvalidBlockRequestTest(BitcoinTestFramework): block3.rehash() block3.solve() - node.p2p.send_blocks_and_test([block3], node, success=False, request_block=False, reject_reason='bad-cb-amount') + node.p2p.send_blocks_and_test([block3], node, success=False, reject_reason='bad-cb-amount') + if __name__ == '__main__': InvalidBlockRequestTest().main() diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index c8c752d1f7..33b7060060 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers +# Copyright (c) 2015-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid locators. diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py new file mode 100755 index 0000000000..700fdf6e04 --- /dev/null +++ b/test/functional/p2p_invalid_messages.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node responses to invalid network messages.""" +import os +import struct + +from test_framework import messages +from test_framework.mininode import P2PDataStore +from test_framework.test_framework import BitcoinTestFramework + + +class msg_unrecognized: + """Nonsensical message. Modeled after similar types in test_framework.messages.""" + + command = b'badmsg' + + def __init__(self, *, str_data): + self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data + + def serialize(self): + return messages.ser_string(self.str_data) + + def __repr__(self): + return "{}(data={})".format(self.command, self.str_data) + + +class InvalidMessagesTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + """ + . Test msg header + 0. Send a bunch of large (4MB) messages of an unrecognized type. Check to see + that it isn't an effective DoS against the node. + + 1. Send an oversized (4MB+) message and check that we're disconnected. + + 2. Send a few messages with an incorrect data size in the header, ensure the + messages are ignored. + """ + self.test_magic_bytes() + self.test_checksum() + self.test_size() + self.test_command() + + node = self.nodes[0] + self.node = node + node.add_p2p_connection(P2PDataStore()) + conn2 = node.add_p2p_connection(P2PDataStore()) + + msg_limit = 4 * 1000 * 1000 # 4MB, per MAX_PROTOCOL_MESSAGE_LENGTH + valid_data_limit = msg_limit - 5 # Account for the 4-byte length prefix + + # + # 0. + # + # Send as large a message as is valid, ensure we aren't disconnected but + # also can't exhaust resources. + # + msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) + assert len(msg_at_size.serialize()) == msg_limit + + increase_allowed = 0.5 + if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]: + increase_allowed = 3.5 + with node.assert_memory_usage_stable(increase_allowed=increase_allowed): + self.log.info( + "Sending a bunch of large, junk messages to test " + "memory exhaustion. May take a bit...") + + # Run a bunch of times to test for memory exhaustion. + for _ in range(80): + node.p2p.send_message(msg_at_size) + + # Check that, even though the node is being hammered by nonsense from one + # connection, it can still service other peers in a timely way. + for _ in range(20): + conn2.sync_with_ping(timeout=2) + + # Peer 1, despite serving up a bunch of nonsense, should still be connected. + self.log.info("Waiting for node to drop junk messages.") + node.p2p.sync_with_ping(timeout=120) + assert node.p2p.is_connected + + # + # 1. + # + # Send an oversized message, ensure we're disconnected. + # + msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1)) + assert len(msg_over_size.serialize()) == (msg_limit + 1) + + with node.assert_debug_log(["Oversized message from peer=4, disconnecting"]): + # An unknown message type (or *any* message type) over + # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. + node.p2p.send_message(msg_over_size) + node.p2p.wait_for_disconnect(timeout=4) + + node.disconnect_p2ps() + conn = node.add_p2p_connection(P2PDataStore()) + conn.wait_for_verack() + + # + # 2. + # + # Send messages with an incorrect data size in the header. + # + actual_size = 100 + msg = msg_unrecognized(str_data="b" * actual_size) + + # TODO: handle larger-than cases. I haven't been able to pin down what behavior to expect. + for wrong_size in (2, 77, 78, 79): + self.log.info("Sending a message with incorrect size of {}".format(wrong_size)) + + # Unmodified message should submit okay. + node.p2p.send_and_ping(msg) + + # A message lying about its data size results in a disconnect when the incorrect + # data size is less than the actual size. + # + # TODO: why does behavior change at 78 bytes? + # + node.p2p.send_raw_message(self._tweak_msg_data_size(msg, wrong_size)) + + # For some reason unknown to me, we sometimes have to push additional data to the + # peer in order for it to realize a disconnect. + try: + node.p2p.send_message(messages.msg_ping(nonce=123123)) + except IOError: + pass + + node.p2p.wait_for_disconnect(timeout=10) + node.disconnect_p2ps() + node.add_p2p_connection(P2PDataStore()) + + # Node is still up. + conn = node.add_p2p_connection(P2PDataStore()) + conn.sync_with_ping() + + def test_magic_bytes(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes + conn.magic_bytes = b'\x00\x11\x22\x32' + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): + conn.send_message(messages.msg_ping(nonce=0xff)) + conn.wait_for_disconnect(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_checksum(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['ProcessMessages(badmsg, 2 bytes): CHECKSUM ERROR expected 78df0a04 was ffffffff']): + msg = conn.build_message(msg_unrecognized(str_data="d")) + cut_len = ( + 4 + # magic + 12 + # command + 4 #len + ) + # modify checksum + msg = msg[:cut_len] + b'\xff' * 4 + msg[cut_len + 4:] + self.nodes[0].p2p.send_raw_message(msg) + conn.sync_with_ping(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_size(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['']): + msg = conn.build_message(msg_unrecognized(str_data="d")) + cut_len = ( + 4 + # magic + 12 # command + ) + # modify len to MAX_SIZE + 1 + msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] + self.nodes[0].p2p.send_raw_message(msg) + conn.wait_for_disconnect(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_command(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): + msg = msg_unrecognized(str_data="d") + msg.command = b'\xff' * 12 + msg = conn.build_message(msg) + # Modify command + msg = msg[:7] + b'\x00' + msg[7 + 1:] + self.nodes[0].p2p.send_raw_message(msg) + conn.sync_with_ping(timeout=1) + self.nodes[0].disconnect_p2ps() + + def _tweak_msg_data_size(self, message, wrong_size): + """ + Return a raw message based on another message but with an incorrect data size in + the message header. + """ + raw_msg = self.node.p2p.build_message(message) + + bad_size_bytes = struct.pack("<I", wrong_size) + num_header_bytes_before_size = 4 + 12 + + # Replace the correct data size in the message with an incorrect one. + raw_msg_with_wrong_size = ( + raw_msg[:num_header_bytes_before_size] + + bad_size_bytes + + raw_msg[(num_header_bytes_before_size + len(bad_size_bytes)):] + ) + assert len(raw_msg) == len(raw_msg_with_wrong_size) + + return raw_msg_with_wrong_size + + +if __name__ == '__main__': + InvalidMessagesTest().main() diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 58e129b57d..1b18dd3e58 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -5,7 +5,7 @@ """Test node responses to invalid transactions. In this test we connect to one node over p2p, and test tx requests.""" -from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script +from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ( COIN, COutPoint, @@ -19,6 +19,7 @@ from test_framework.util import ( assert_equal, wait_until, ) +from data import invalid_txs class InvalidTxRequestTest(BitcoinTestFramework): @@ -63,12 +64,21 @@ class InvalidTxRequestTest(BitcoinTestFramework): self.log.info("Mature the block.") self.nodes[0].generatetoaddress(100, self.nodes[0].get_deterministic_priv_key().address) - # b'\x64' is OP_NOTIF - # Transaction will be rejected with code 16 (REJECT_INVALID) - # and we get disconnected immediately - self.log.info('Test a transaction that is rejected') - tx1 = create_tx_with_script(block1.vtx[0], 0, script_sig=b'\x64' * 35, amount=50 * COIN - 12000) - node.p2p.send_txs_and_test([tx1], node, success=False, expect_disconnect=True) + # Iterate through a list of known invalid transaction types, ensuring each is + # rejected. Some are consensus invalid and some just violate policy. + for BadTxTemplate in invalid_txs.iter_all_templates(): + self.log.info("Testing invalid transaction: %s", BadTxTemplate.__name__) + template = BadTxTemplate(spend_block=block1) + tx = template.get_tx() + node.p2p.send_txs_and_test( + [tx], node, success=False, + expect_disconnect=template.expect_disconnect, + reject_reason=template.reject_reason, + ) + + if template.expect_disconnect: + self.log.info("Reconnecting to peer") + self.reconnect_p2p() # Make two p2p connections to provide the node with orphans # * p2ps[0] will send valid orphan txs (one with low fee) diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py new file mode 100755 index 0000000000..dc4d475b2d --- /dev/null +++ b/test/functional/p2p_leak_tx.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that we don't leak txs to inbound peers that we haven't yet announced to""" + +from test_framework.messages import msg_getdata, CInv +from test_framework.mininode import P2PDataStore +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class P2PNode(P2PDataStore): + def on_inv(self, msg): + pass + + +class P2PLeakTxTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + gen_node = self.nodes[0] # The block and tx generating node + gen_node.generate(1) + + inbound_peer = self.nodes[0].add_p2p_connection(P2PNode()) # An "attacking" inbound peer + + MAX_REPEATS = 100 + self.log.info("Running test up to {} times.".format(MAX_REPEATS)) + for i in range(MAX_REPEATS): + self.log.info('Run repeat {}'.format(i + 1)) + txid = gen_node.sendtoaddress(gen_node.getnewaddress(), 0.01) + + want_tx = msg_getdata() + want_tx.inv.append(CInv(t=1, h=int(txid, 16))) + inbound_peer.last_message.pop('notfound', None) + inbound_peer.send_message(want_tx) + inbound_peer.sync_with_ping() + + if inbound_peer.last_message.get('notfound'): + self.log.debug('tx {} was not yet announced to us.'.format(txid)) + self.log.debug("node has responded with a notfound message. End test.") + assert_equal(inbound_peer.last_message['notfound'].vec[0].hash, int(txid, 16)) + inbound_peer.last_message.pop('notfound') + break + else: + self.log.debug('tx {} was already announced to us. Try test again.'.format(txid)) + assert int(txid, 16) in [inv.hash for inv in inbound_peer.last_message['inv'].inv] + + +if __name__ == '__main__': + P2PLeakTxTest().main() diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index ec3d336dc1..359880506e 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -43,8 +43,8 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): disconnect_nodes(self.nodes[1], 2) def setup_network(self): - super(NodeNetworkLimitedTest, self).setup_network() - self.disconnect_all() + self.add_nodes(self.num_nodes, self.extra_args) + self.start_nodes() def run_test(self): node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index afbbfa8992..8f8e89cf15 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -545,31 +545,13 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_getblocktemplate_before_lockin(self): - # Node0 is segwit aware, node2 is not. - for node in [self.nodes[0], self.nodes[2]]: - gbt_results = node.getblocktemplate() - block_version = gbt_results['version'] - # If we're not indicating segwit support, we will still be - # signalling for segwit activation. - assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0]) - # If we don't specify the segwit rule, then we won't get a default - # commitment. - assert('default_witness_commitment' not in gbt_results) - - # Workaround: - # Can either change the tip, or change the mempool and wait 5 seconds - # to trigger a recomputation of getblocktemplate. txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) - # Using mocktime lets us avoid sleep() - sync_mempools(self.nodes) - self.nodes[0].setmocktime(int(time.time()) + 10) - self.nodes[2].setmocktime(int(time.time()) + 10) for node in [self.nodes[0], self.nodes[2]]: gbt_results = node.getblocktemplate({"rules": ["segwit"]}) block_version = gbt_results['version'] if node == self.nodes[2]: - # If this is a non-segwit node, we should still not get a witness + # If this is a non-segwit node, we should not get a witness # commitment, nor a version bit signalling segwit. assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) assert('default_witness_commitment' not in gbt_results) @@ -586,10 +568,6 @@ class SegWitTest(BitcoinTestFramework): script = get_witness_script(witness_root, 0) assert_equal(witness_commitment, bytes_to_hex_str(script)) - # undo mocktime - self.nodes[0].setmocktime(0) - self.nodes[2].setmocktime(0) - @subtest def advance_to_segwit_lockin(self): """Mine enough blocks to lock in segwit, but don't activate.""" @@ -777,7 +755,7 @@ class SegWitTest(BitcoinTestFramework): spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) spend_tx.rehash() with self.nodes[0].assert_debug_log( - expected_msgs=('Not relaying invalid transaction {}'.format(spend_tx.hash), 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')): + expected_msgs=(spend_tx.hash, 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')): test_transaction_acceptance(self.nodes[0], self.test_node, spend_tx, with_witness=False, accepted=False) # Now put the witness script in the witness, should succeed after diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 2459a9f243..02ceec3dc1 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -14,11 +14,11 @@ - Wait 1 second - Assert that we're connected - Send a ping to no_verack_node and no_version_node -- Wait 30 seconds +- Wait 1 second - Assert that we're still connected - Send a ping to no_verack_node and no_version_node -- Wait 31 seconds -- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds) +- Wait 2 seconds +- Assert that we're no longer connected (timeout to receive version/verack is 3 seconds) """ from time import sleep @@ -36,6 +36,8 @@ class TimeoutsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + # set timeout to receive version/verack to 3 seconds + self.extra_args = [["-peertimeout=3"]] def run_test(self): # Setup the p2p connections @@ -52,7 +54,7 @@ class TimeoutsTest(BitcoinTestFramework): no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) - sleep(30) + sleep(1) assert "version" in no_verack_node.last_message @@ -63,11 +65,21 @@ class TimeoutsTest(BitcoinTestFramework): no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) - sleep(31) - - assert not no_verack_node.is_connected - assert not no_version_node.is_connected - assert not no_send_node.is_connected + expected_timeout_logs = [ + "version handshake timeout from 0", + "socket no message in first 3 seconds, 1 0 from 1", + "socket no message in first 3 seconds, 0 0 from 2", + ] + + with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): + sleep(3) + # By now, we waited a total of 5 seconds. Off-by-two for two + # reasons: + # * The internal precision is one second + # * Account for network delay + assert not no_verack_node.is_connected + assert not no_version_node.is_connected + assert not no_send_node.is_connected if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 53916d5290..3938ca98dd 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -48,9 +48,12 @@ class RPCBindTest(BitcoinTestFramework): at a non-localhost IP. ''' self.log.info("Allow IP test for %s:%d" % (rpchost, rpcport)) - base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] + node_args = \ + ['-disablewallet', '-nolisten'] + \ + ['-rpcallowip='+x for x in allow_ips] + \ + ['-rpcbind='+addr for addr in ['127.0.0.1', "%s:%d" % (rpchost, rpcport)]] # Bind to localhost as well so start_nodes doesn't hang self.nodes[0].rpchost = None - self.start_nodes([base_args]) + self.start_nodes([node_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(self.nodes[0].datadir, 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir) node.getnetworkinfo() @@ -67,7 +70,7 @@ class RPCBindTest(BitcoinTestFramework): self.log.info("Check for ipv6") have_ipv6 = test_ipv6_local() - if not have_ipv6 and not self.options.run_ipv4: + if not have_ipv6 and not (self.options.run_ipv4 or self.options.run_nonloopback): raise SkipTest("This test requires ipv6 support.") self.log.info("Check for non-loopback interface") @@ -101,9 +104,9 @@ class RPCBindTest(BitcoinTestFramework): # check default without rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(None, '127.0.0.1', [], [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) - # check default with rpcallowip (IPv6 any) + # check default with rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], - [('::0', self.defaultport)]) + [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) # check only IPv6 localhost (explicit) self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], [('::1', self.defaultport)]) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 92b690176d..31e60f1cea 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -133,7 +133,7 @@ class BlockchainTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Block is not in main chain", self.nodes[0].getchaintxstats, blockhash=blockhash) self.nodes[0].reconsiderblock(blockhash) - chaintxstats = self.nodes[0].getchaintxstats(1) + chaintxstats = self.nodes[0].getchaintxstats(nblocks=1) # 200 txs plus genesis tx assert_equal(chaintxstats['txcount'], 201) # tx rate should be 1 per 10 minutes, or 1/600 @@ -211,7 +211,7 @@ class BlockchainTest(BitcoinTestFramework): besthash = node.getbestblockhash() secondbesthash = node.getblockhash(199) - header = node.getblockheader(besthash) + header = node.getblockheader(blockhash=besthash) assert_equal(header['hash'], besthash) assert_equal(header['height'], 200) @@ -287,7 +287,7 @@ class BlockchainTest(BitcoinTestFramework): def assert_waitforheight(height, timeout=2): assert_equal( - node.waitforblockheight(height, timeout)['height'], + node.waitforblockheight(height=height, timeout=timeout)['height'], current_height) assert_waitforheight(0) diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 58074803cc..588bfbe083 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -4,12 +4,17 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], ["-deprecatedrpc=validateaddress"]] + self.extra_args = [[], ["-deprecatedrpc=generate"]] + + def skip_test_if_missing_module(self): + # The generate RPC method requires the wallet to be compiled + self.skip_if_no_wallet() def run_test(self): # This test should be used to verify correct behaviour of deprecated @@ -18,7 +23,10 @@ class DeprecatedRpcTest(BitcoinTestFramework): # self.log.info("Make sure that -deprecatedrpc=createmultisig allows it to take addresses") # assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, [self.nodes[0].getnewaddress()]) # self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) - pass + + self.log.info("Test generate RPC") + assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) + self.nodes[1].generate(1) if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py new file mode 100755 index 0000000000..2cc5bc974b --- /dev/null +++ b/test/functional/rpc_deriveaddresses.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the deriveaddresses rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +class DeriveaddressesTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.supports_cli = 1 + + def run_test(self): + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a") + + descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + + assert_equal(self.nodes[0].deriveaddresses(descriptor), [address]) + + descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + + assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) + + ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)" + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 0, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)", 0, 2) + + assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)") + + assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 0) + + assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 2, 0) + + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", -1, 0) + + combo_descriptor = "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) + + hardened_without_privkey_descriptor = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)" + assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) + + bare_multisig_descriptor = "multi(1, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)" + assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor) + +if __name__ == '__main__': + DeriveaddressesTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 0c61e9ab62..4f350953b2 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -32,7 +32,7 @@ class RawTransactionsTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.setup_nodes() connect_nodes_bi(self.nodes, 0, 1) diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index b24bed6adf..ca9e24367a 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -170,6 +170,8 @@ class GetblockstatsTest(BitcoinTestFramework): self.nodes[0].getblockstats, hash_or_height=1, stats=['minfee' , 'aaa%s' % inv_sel_stat]) assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled', + self.nodes[1].getblockstats, hash_or_height=1) + assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled', self.nodes[1].getblockstats, hash_or_height=self.start_height + self.max_stat_pos) # Mainchain's genesis block shouldn't be found on regtest diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index be096af892..78d6e78aed 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -7,12 +7,18 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +import os + class HelpRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 def run_test(self): + self.test_categories() + self.dump_help() + + def test_categories(self): node = self.nodes[0] # wrong argument count @@ -37,6 +43,15 @@ class HelpRpcTest(BitcoinTestFramework): assert_equal(titles, components) + def dump_help(self): + dump_dir = os.path.join(self.options.tmpdir, 'rpc_help_dump') + os.mkdir(dump_dir) + calls = [line.split(' ', 1)[0] for line in self.nodes[0].help().splitlines() if line and not line.startswith('==')] + for call in calls: + with open(os.path.join(dump_dir, call), 'w', encoding='utf-8') as f: + # Make sure the node can generate the help at runtime without crashing + f.write(self.nodes[0].help(call)) + if __name__ == '__main__': HelpRpcTest().main() diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index d8ecdd573a..d8a1deb2a3 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -4,10 +4,15 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" -import time - from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, connect_nodes_bi, sync_blocks +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.util import ( + assert_equal, + connect_nodes_bi, + sync_blocks, + wait_until, +) + class InvalidateTest(BitcoinTestFramework): def set_test_params(self): @@ -21,46 +26,66 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") self.log.info("Mine 4 blocks on Node 0") self.nodes[0].generatetoaddress(4, self.nodes[0].get_deterministic_priv_key().address) - assert(self.nodes[0].getblockcount() == 4) - besthash = self.nodes[0].getbestblockhash() + assert_equal(self.nodes[0].getblockcount(), 4) + besthash_n0 = self.nodes[0].getbestblockhash() self.log.info("Mine competing 6 blocks on Node 1") self.nodes[1].generatetoaddress(6, self.nodes[1].get_deterministic_priv_key().address) - assert(self.nodes[1].getblockcount() == 6) + assert_equal(self.nodes[1].getblockcount(), 6) self.log.info("Connect nodes to force a reorg") - connect_nodes_bi(self.nodes,0,1) + connect_nodes_bi(self.nodes, 0, 1) sync_blocks(self.nodes[0:2]) - assert(self.nodes[0].getblockcount() == 6) + assert_equal(self.nodes[0].getblockcount(), 6) badhash = self.nodes[1].getblockhash(2) self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain") self.nodes[0].invalidateblock(badhash) - newheight = self.nodes[0].getblockcount() - newhash = self.nodes[0].getbestblockhash() - if (newheight != 4 or newhash != besthash): - raise AssertionError("Wrong tip for node0, hash %s, height %d"%(newhash,newheight)) + assert_equal(self.nodes[0].getblockcount(), 4) + assert_equal(self.nodes[0].getbestblockhash(), besthash_n0) self.log.info("Make sure we won't reorg to a lower work chain:") - connect_nodes_bi(self.nodes,1,2) + connect_nodes_bi(self.nodes, 1, 2) self.log.info("Sync node 2 to node 1 so both have 6 blocks") sync_blocks(self.nodes[1:3]) - assert(self.nodes[2].getblockcount() == 6) + assert_equal(self.nodes[2].getblockcount(), 6) self.log.info("Invalidate block 5 on node 1 so its tip is now at 4") self.nodes[1].invalidateblock(self.nodes[1].getblockhash(5)) - assert(self.nodes[1].getblockcount() == 4) + assert_equal(self.nodes[1].getblockcount(), 4) self.log.info("Invalidate block 3 on node 2, so its tip is now 2") self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3)) - assert(self.nodes[2].getblockcount() == 2) + assert_equal(self.nodes[2].getblockcount(), 2) self.log.info("..and then mine a block") self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") - time.sleep(5) - assert_equal(self.nodes[2].getblockcount(), 3) - assert_equal(self.nodes[0].getblockcount(), 4) - node1height = self.nodes[1].getblockcount() - if node1height < 4: - raise AssertionError("Node 1 reorged to a lower height: %d"%node1height) + wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) + wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) + wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) + + self.log.info("Verify that we reconsider all ancestors as well") + blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + # Invalidate the two blocks at the tip + self.nodes[1].invalidateblock(blocks[-1]) + self.nodes[1].invalidateblock(blocks[-2]) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-3]) + # Reconsider only the previous tip + self.nodes[1].reconsiderblock(blocks[-1]) + # Should be back at the tip by now + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + + self.log.info("Verify that we reconsider all descendants") + blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + # Invalidate the two blocks at the tip + self.nodes[1].invalidateblock(blocks[-2]) + self.nodes[1].invalidateblock(blocks[-4]) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-5]) + # Reconsider only the previous tip + self.nodes[1].reconsiderblock(blocks[-4]) + # Should be back at the tip by now + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + if __name__ == '__main__': InvalidateTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 1e525214fa..b12eb1d9ec 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -67,19 +67,19 @@ class NetTest(BitcoinTestFramework): peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): - assert_greater_than_or_equal(after['bytesrecv_per_msg']['pong'], before['bytesrecv_per_msg']['pong'] + 32) - assert_greater_than_or_equal(after['bytessent_per_msg']['ping'], before['bytessent_per_msg']['ping'] + 32) + assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32) + assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32) def _test_getnetworkinginfo(self): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) - self.nodes[0].setnetworkactive(False) + self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) - self.nodes[0].setnetworkactive(True) + self.nodes[0].setnetworkactive(state=True) connect_nodes_bi(self.nodes, 0, 1) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) @@ -88,7 +88,7 @@ class NetTest(BitcoinTestFramework): assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) - self.nodes[0].addnode(ip_port, 'add') + self.nodes[0].addnode(node=ip_port, command='add') # check that the node has indeed been added added_nodes = self.nodes[0].getaddednodeinfo(ip_port) assert_equal(len(added_nodes), 1) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 54dc871448..a82a5d0208 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -6,7 +6,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, find_output +from test_framework.util import assert_equal, assert_raises_rpc_error, find_output, disconnect_nodes, connect_nodes_bi, sync_blocks import json import os @@ -19,10 +19,51 @@ class PSBTTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 3 + # TODO: remove -txindex. Currently required for getrawtransaction call. + self.extra_args = [[], ["-txindex"], ["-txindex"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def test_utxo_conversion(self): + mining_node = self.nodes[2] + offline_node = self.nodes[0] + online_node = self.nodes[1] + + # Disconnect offline node from others + disconnect_nodes(offline_node, 1) + disconnect_nodes(online_node, 0) + disconnect_nodes(offline_node, 2) + disconnect_nodes(mining_node, 0) + + # Mine a transaction that credits the offline address + offline_addr = offline_node.getnewaddress(address_type="p2sh-segwit") + online_addr = online_node.getnewaddress(address_type="p2sh-segwit") + online_node.importaddress(offline_addr, "", False) + mining_node.sendtoaddress(address=offline_addr, amount=1.0) + mining_node.generate(nblocks=1) + sync_blocks([mining_node, online_node]) + + # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) + utxos = online_node.listunspent(addresses=[offline_addr]) + raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) + psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] + assert("non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]) + + # Have the offline node sign the PSBT (which will update the UTXO to segwit) + signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] + assert("witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0]) + + # Make sure we can mine the resulting transaction + txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) + mining_node.generate(1) + sync_blocks([mining_node, online_node]) + assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) + + # Reconnect + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 0, 2) + def run_test(self): # Create and fund a raw tx for sending 10 BTC psbtx1 = self.nodes[0].walletcreatefundedpsbt([], {self.nodes[2].getnewaddress():10})['psbt'] @@ -107,6 +148,9 @@ class PSBTTest(BitcoinTestFramework): # Make sure that a psbt with signatures cannot be converted signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex']) assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex']) + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex'], False) + # Unless we allow it to convert and strip signatures + self.nodes[0].converttopsbt(signedtx['hex'], True) # Explicitly allow converting non-empty txs new_psbt = self.nodes[0].converttopsbt(rawtx['hex']) @@ -168,6 +212,17 @@ class PSBTTest(BitcoinTestFramework): assert tx_in["sequence"] > MAX_BIP125_RBF_SEQUENCE assert_equal(decoded_psbt["tx"]["locktime"], 0) + # Make sure change address wallet does not have P2SH innerscript access to results in success + # when attempting BnB coin selection + self.nodes[0].walletcreatefundedpsbt([], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"changeAddress":self.nodes[1].getnewaddress()}, False) + + # Regression test for 14473 (mishandling of already-signed witness transaction): + psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}]) + complete_psbt = self.nodes[0].walletprocesspsbt(psbtx_info["psbt"]) + double_processed_psbt = self.nodes[0].walletprocesspsbt(complete_psbt["psbt"]) + assert_equal(complete_psbt, double_processed_psbt) + # We don't care about the decode result, but decoding must succeed. + self.nodes[0].decodepsbt(double_processed_psbt["psbt"]) # BIP 174 Test Vectors @@ -214,6 +269,9 @@ class PSBTTest(BitcoinTestFramework): combined = self.nodes[2].combinepsbt(combiner['combine']) assert_equal(combined, combiner['result']) + # Empty combiner test + assert_raises_rpc_error(-8, "Parameter 'txs' cannot be empty", self.nodes[0].combinepsbt, []) + # Finalizer test for finalizer in finalizers: finalized = self.nodes[2].finalizepsbt(finalizer['finalize'], False)['psbt'] @@ -224,6 +282,16 @@ class PSBTTest(BitcoinTestFramework): extracted = self.nodes[2].finalizepsbt(extractor['extract'], True)['hex'] assert_equal(extracted, extractor['result']) + # Unload extra wallets + for i, signer in enumerate(signers): + self.nodes[2].unloadwallet("wallet{}".format(i)) + + self.test_utxo_conversion() + + # Test that psbts with p2pkh outputs are created properly + p2pkh = self.nodes[0].getnewaddress(address_type='legacy') + psbt = self.nodes[1].walletcreatefundedpsbt([], [{p2pkh : 1}], 0, {"includeWatching" : True}, True) + self.nodes[0].decodepsbt(psbt['psbt']) if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 8ed490f552..a97d753626 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -42,12 +42,13 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-addresstype=legacy"], ["-addresstype=legacy"], ["-addresstype=legacy"]] + # TODO: remove -txindex. Currently required for getrawtransaction call. + self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): super().setup_network() connect_nodes_bi(self.nodes, 0, 2) @@ -100,6 +101,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].createrawtransaction, [], {address: -1}) assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], multidict([(address, 1), (address, 1)])) assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], [{address: 1}, {address: 1}]) + assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], [{"data": 'aa'}, {"data": "bb"}]) + assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], multidict([("data", 'aa'), ("data", "bb")])) assert_raises_rpc_error(-8, "Invalid parameter, key-value pair must contain exactly one key", self.nodes[0].createrawtransaction, [], [{'a': 1, 'b': 2}]) assert_raises_rpc_error(-8, "Invalid parameter, key-value pair not an object as expected", self.nodes[0].createrawtransaction, [], [['key-value pair1'], ['2']]) @@ -127,19 +130,12 @@ class RawTransactionsTest(BitcoinTestFramework): bytes_to_hex_str(tx.serialize()), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) - # Two data outputs - tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([('data', '99'), ('data', '99')]))))) - assert_equal(len(tx.vout), 2) - assert_equal( - bytes_to_hex_str(tx.serialize()), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{'data': '99'}, {'data': '99'}]), - ) # Multiple mixed outputs - tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), ('data', '99'), ('data', '99')]))))) + tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))))) assert_equal(len(tx.vout), 3) assert_equal( bytes_to_hex_str(tx.serialize()), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {'data': '99'}, {'data': '99'}]), + self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) for type in ["bech32", "p2sh-segwit", "legacy"]: diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 96f9ccdbdb..11b4db6ec5 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -10,6 +10,9 @@ from decimal import Decimal import shutil import os +def descriptors(out): + return sorted(u['desc'] for u in out['unspents']) + class ScantxoutsetTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -65,6 +68,8 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ "addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "combo(" + pubk3 + ")"])['total_amount'], Decimal("0.007")) self.log.info("Test extended key derivation.") + # Run various scans, and verify that the sum of the amounts of the matches corresponds to the expected subset. + # Note that all amounts in the UTXO set are powers of 2 multiplied by 0.001 BTC, so each amounts uniquely identifies a subset. assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0'/0h/0h)"])['total_amount'], Decimal("0.008")) assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0'/0'/1h)"])['total_amount'], Decimal("0.016")) assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/1500')"])['total_amount'], Decimal("0.032")) @@ -82,7 +87,7 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/1)"])['total_amount'], Decimal("8.192")) assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/1500)"])['total_amount'], Decimal("16.384")) assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)"])['total_amount'], Decimal("4.096")) - assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)"])['total_amount'], Decimal("8.192")) + assert_equal(self.nodes[0].scantxoutset("start", [ "combo([abcdef88/1/2'/3/4h]tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)"])['total_amount'], Decimal("8.192")) assert_equal(self.nodes[0].scantxoutset("start", [ "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1500)"])['total_amount'], Decimal("16.384")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*')", "range": 1499}])['total_amount'], Decimal("1.536")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*')", "range": 1500}])['total_amount'], Decimal("3.584")) @@ -91,5 +96,10 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1499}])['total_amount'], Decimal("12.288")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) + # Test the reported descriptors for a few matches + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)']) + if __name__ == '__main__': ScantxoutsetTest().main() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 35004fb588..6b47cae4c3 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -41,12 +41,15 @@ from .script import ( from .util import assert_equal from io import BytesIO +MAX_BLOCK_SIGOPS = 20000 + # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" -def create_block(hashprev, coinbase, ntime=None): +def create_block(hashprev, coinbase, ntime=None, *, version=1): """Create a block (with regtest difficulty).""" block = CBlock() + block.nVersion = version if ntime is None: import time block.nTime = int(time.time() + 600) @@ -169,7 +172,7 @@ def get_legacy_sigopcount_tx(tx, accurate=True): return count def witness_script(use_p2wsh, pubkey): - """Create a scriptPubKey for a pay-to-wtiness TxOut. + """Create a scriptPubKey for a pay-to-witness TxOut. This is either a P2WPKH output for the given pubkey, or a P2WSH output of a 1-of-1 multisig for the given pubkey. Returns the hex encoding of the diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8e9372767d..356a45d6d0 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -35,7 +35,6 @@ MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) -MAX_INV_SZ = 50000 MAX_LOCATOR_SZ = 101 MAX_BLOCK_BASE_SIZE = 1000000 @@ -58,9 +57,6 @@ MSG_TYPE_MASK = 0xffffffff >> 2 def sha256(s): return hashlib.new('sha256', s).digest() -def ripemd160(s): - return hashlib.new('ripemd160', s).digest() - def hash256(s): return sha256(sha256(s)) @@ -454,6 +450,8 @@ class CTransaction: if flags != 0: self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] self.wit.deserialize(f) + else: + self.wit = CTxWitness() self.nLockTime = struct.unpack("<I", f.read(4))[0] self.sha256 = None self.hash = None @@ -768,7 +766,7 @@ class HeaderAndShortIDs: self.prefilled_txn = [] self.use_witness = False - if p2pheaders_and_shortids != None: + if p2pheaders_and_shortids is not None: self.header = p2pheaders_and_shortids.header self.nonce = p2pheaders_and_shortids.nonce self.shortids = p2pheaders_and_shortids.shortids @@ -826,7 +824,7 @@ class BlockTransactionsRequest: def __init__(self, blockhash=0, indexes = None): self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] + self.indexes = indexes if indexes is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -867,7 +865,7 @@ class BlockTransactions: def __init__(self, blockhash=0, transactions = None): self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] + self.transactions = transactions if transactions is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -887,13 +885,12 @@ class BlockTransactions: class CPartialMerkleTree: - __slots__ = ("fBad", "nTransactions", "vBits", "vHash") + __slots__ = ("nTransactions", "vBits", "vHash") def __init__(self): self.nTransactions = 0 self.vHash = [] self.vBits = [] - self.fBad = False def deserialize(self, f): self.nTransactions = struct.unpack("<i", f.read(4))[0] @@ -1057,7 +1054,7 @@ class msg_getdata: command = b"getdata" def __init__(self, inv=None): - self.inv = inv if inv != None else [] + self.inv = inv if inv is not None else [] def deserialize(self, f): self.inv = deser_vector(f, CInv) @@ -1232,6 +1229,23 @@ class msg_mempool: return "msg_mempool()" +class msg_notfound: + __slots__ = ("vec", ) + command = b"notfound" + + def __init__(self, vec=None): + self.vec = vec or [] + + def deserialize(self, f): + self.vec = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.vec) + + def __repr__(self): + return "msg_notfound(vec=%s)" % (repr(self.vec)) + + class msg_sendheaders: __slots__ = () command = b"sendheaders" diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 6864e8a838..ac7cc068bd 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -21,7 +21,38 @@ import struct import sys import threading -from test_framework.messages import CBlockHeader, MIN_VERSION_SUPPORTED, msg_addr, msg_block, MSG_BLOCK, msg_blocktxn, msg_cmpctblock, msg_feefilter, msg_getaddr, msg_getblocks, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_mempool, msg_ping, msg_pong, msg_reject, msg_sendcmpct, msg_sendheaders, msg_tx, MSG_TX, MSG_TYPE_MASK, msg_verack, msg_version, NODE_NETWORK, NODE_WITNESS, sha256 +from test_framework.messages import ( + CBlockHeader, + MIN_VERSION_SUPPORTED, + msg_addr, + msg_block, + MSG_BLOCK, + msg_blocktxn, + msg_cmpctblock, + msg_feefilter, + msg_getaddr, + msg_getblocks, + msg_getblocktxn, + msg_getdata, + msg_getheaders, + msg_headers, + msg_inv, + msg_mempool, + msg_notfound, + msg_ping, + msg_pong, + msg_reject, + msg_sendcmpct, + msg_sendheaders, + msg_tx, + MSG_TX, + MSG_TYPE_MASK, + msg_verack, + msg_version, + NODE_NETWORK, + NODE_WITNESS, + sha256, +) from test_framework.util import wait_until logger = logging.getLogger("TestFramework.mininode") @@ -40,6 +71,7 @@ MESSAGEMAP = { b"headers": msg_headers, b"inv": msg_inv, b"mempool": msg_mempool, + b"notfound": msg_notfound, b"ping": msg_ping, b"pong": msg_pong, b"reject": msg_reject, @@ -86,7 +118,7 @@ class P2PConnection(asyncio.Protocol): # The initial message to send after the connection was made: self.on_connection_send_msg = None self.recvbuf = b"" - self.network = net + self.magic_bytes = MAGIC_BYTES[net] logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) loop = NetworkThread.network_event_loop @@ -138,7 +170,7 @@ class P2PConnection(asyncio.Protocol): while True: if len(self.recvbuf) < 4: return - if self.recvbuf[:4] != MAGIC_BYTES[self.network]: + if self.recvbuf[:4] != self.magic_bytes: raise ValueError("got garbage %s" % repr(self.recvbuf)) if len(self.recvbuf) < 4 + 12 + 4 + 4: return @@ -175,10 +207,13 @@ class P2PConnection(asyncio.Protocol): This method takes a P2P payload, builds the P2P header and adds the message to the send buffer to be sent over the socket.""" + tmsg = self.build_message(message) + self._log_message("send", message) + return self.send_raw_message(tmsg) + + def send_raw_message(self, raw_message_bytes): if not self.is_connected: raise IOError('Not connected') - self._log_message("send", message) - tmsg = self._build_message(message) def maybe_write(): if not self._transport: @@ -188,16 +223,16 @@ class P2PConnection(asyncio.Protocol): # Python 3.4 versions. if hasattr(self._transport, 'is_closing') and self._transport.is_closing(): return - self._transport.write(tmsg) + self._transport.write(raw_message_bytes) NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write) # Class utility methods - def _build_message(self, message): + def build_message(self, message): """Build a serialized P2P message""" command = message.command data = message.serialize() - tmsg = MAGIC_BYTES[self.network] + tmsg = self.magic_bytes tmsg += command tmsg += b"\x00" * (12 - len(command)) tmsg += struct.pack("<I", len(data)) @@ -295,6 +330,7 @@ class P2PInterface(P2PConnection): def on_getheaders(self, message): pass def on_headers(self, message): pass def on_mempool(self, message): pass + def on_notfound(self, message): pass def on_pong(self, message): pass def on_reject(self, message): pass def on_sendcmpct(self, message): pass @@ -313,7 +349,7 @@ class P2PInterface(P2PConnection): self.send_message(msg_pong(message.nonce)) def on_verack(self, message): - self.verack_received = True + pass def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) @@ -376,9 +412,9 @@ class P2PInterface(P2PConnection): # Message sending helper functions - def send_and_ping(self, message): + def send_and_ping(self, message, timeout=60): self.send_message(message) - self.sync_with_ping() + self.sync_with_ping(timeout=timeout) # Sync up with the node def sync_with_ping(self, timeout=60): @@ -475,14 +511,14 @@ class P2PDataStore(P2PInterface): if response is not None: self.send_message(response) - def send_blocks_and_test(self, blocks, node, *, success=True, request_block=True, reject_reason=None, expect_disconnect=False, timeout=60): + def send_blocks_and_test(self, blocks, node, *, success=True, force_send=False, reject_reason=None, expect_disconnect=False, timeout=60): """Send blocks to test node and test whether the tip advances. - add all blocks to our block_store - send a headers message for the final block - the on_getheaders handler will ensure that any getheaders are responded to - - if request_block is True: wait for getdata for each of the blocks. The on_getdata handler will - ensure that any getdata messages are responded to + - if force_send is False: wait for getdata for each of the blocks. The on_getdata handler will + ensure that any getdata messages are responded to. Otherwise send the full block unsolicited. - if success is True: assert that the node's tip advances to the most recent block - if success is False: assert that the node's tip doesn't advance - if reject_reason is set: assert that the correct reject message is logged""" @@ -494,9 +530,11 @@ class P2PDataStore(P2PInterface): reject_reason = [reject_reason] if reject_reason else [] with node.assert_debug_log(expected_msgs=reject_reason): - self.send_message(msg_headers([CBlockHeader(blocks[-1])])) - - if request_block: + if force_send: + for b in blocks: + self.send_message(msg_block(block=b)) + else: + self.send_message(msg_headers([CBlockHeader(blocks[-1])])) wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) if expect_disconnect: diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 2fe44010ba..012c80a1be 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -385,6 +385,22 @@ class CScriptNum: r[-1] |= 0x80 return bytes([len(r)]) + r + @staticmethod + def decode(vch): + result = 0 + # We assume valid push_size and minimal encoding + value = vch[1:] + if len(value) == 0: + return result + for i, byte in enumerate(value): + result |= int(byte) << 8*i + if value[-1] >= 0x80: + # Mask for all but the highest result bit + num_mask = (2**(len(value)*8) - 1) >> 1 + result &= num_mask + result *= -1 + return result + class CScript(bytes): """Serialized script @@ -434,6 +450,10 @@ class CScript(bytes): # join makes no sense for a CScript() raise NotImplementedError + # Python 3.4 compatibility + def hex(self): + return hexlify(self).decode('ascii') + def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): return super(CScript, cls).__new__(cls, value) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index dd0f209268..a21c864e75 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -54,10 +54,9 @@ class Socks5Command(): return 'Socks5Command(%s,%s,%s,%s,%s,%s)' % (self.cmd, self.atyp, self.addr, self.port, self.username, self.password) class Socks5Connection(): - def __init__(self, serv, conn, peer): + def __init__(self, serv, conn): self.serv = serv self.conn = conn - self.peer = peer def handle(self): """Handle socks5 request according to RFC192.""" @@ -137,9 +136,9 @@ class Socks5Server(): def run(self): while self.running: - (sockconn, peer) = self.s.accept() + (sockconn, _) = self.s.accept() if self.running: - conn = Socks5Connection(self, sockconn, peer) + conn = Socks5Connection(self, sockconn) thread = threading.Thread(None, conn.handle) thread.daemon = True thread.start() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 7e2ec673df..8c4c0d7226 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -43,6 +43,8 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 +TMPDIR_PREFIX = "bitcoin_func_test_" + class SkipTest(Exception): """This exception is raised to skip a test""" @@ -93,7 +95,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.nodes = [] self.network_thread = None self.mocktime = 0 - self.rpc_timewait = 60 # Wait for up to 60 seconds for the RPC server to respond + self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = False self.bind_to_localhost_only = True self.set_test_params() @@ -126,6 +128,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use bitcoin-cli instead of RPC for all commands") + parser.add_argument("--perf", dest="perf", default=False, action="store_true", + help="profile running nodes with perf for the duration of the test") self.add_options(parser) self.options = parser.parse_args() @@ -137,6 +141,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) + self.config = config self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) @@ -151,7 +156,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: - self.options.tmpdir = tempfile.mkdtemp(prefix="test") + self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX) self._start_logging() self.log.debug('Setting up network thread') @@ -168,7 +173,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.skip_test_if_missing_module() self.setup_chain() self.setup_network() - self.import_deterministic_coinbase_privkeys() self.run_test() success = TestStatus.PASSED except JSONRPCException as e: @@ -200,11 +204,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): node.cleanup_on_exit = False self.log.info("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: + should_clean_up = ( + not self.options.nocleanup and + not self.options.noshutdown and + success != TestStatus.FAILED and + not self.options.perf + ) + if should_clean_up: self.log.info("Cleaning up {} on exit".format(self.options.tmpdir)) cleanup_tree_on_exit = True + elif self.options.perf: + self.log.warning("Not cleaning up dir {} due to perf data".format(self.options.tmpdir)) + cleanup_tree_on_exit = False else: - self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) + self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False if success == TestStatus.PASSED: @@ -261,11 +274,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() + self.import_deterministic_coinbase_privkeys() def import_deterministic_coinbase_privkeys(self): - if self.setup_clean_chain: - return - for n in self.nodes: try: n.getwalletinfo() @@ -273,7 +284,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert str(e).startswith('Method not found') continue - n.importprivkey(n.get_deterministic_priv_key().key) + n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') def run_test(self): """Tests must override this method to define test logic""" @@ -282,7 +293,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Public helper methods. These can be accessed by the subclass test scripts. def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None): - """Instantiate TestNode objects""" + """Instantiate TestNode objects. + + Should only be called once after the nodes have been specified in + set_test_params().""" if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -295,7 +309,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, get_datadir_path(self.options.tmpdir, i), rpchost=rpchost, timewait=self.rpc_timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli)) + self.nodes.append(TestNode( + i, + get_datadir_path(self.options.tmpdir, i), + rpchost=rpchost, + timewait=self.rpc_timeout, + bitcoind=binary[i], + bitcoin_cli=self.options.bitcoincli, + mocktime=self.mocktime, + coverage_dir=self.options.coveragedir, + extra_conf=extra_confs[i], + extra_args=extra_args[i], + use_cli=self.options.usecli, + start_perf=self.options.perf, + )) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" @@ -328,16 +355,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) - def stop_node(self, i, expected_stderr=''): + def stop_node(self, i, expected_stderr='', wait=0): """Stop a bitcoind test node""" - self.nodes[i].stop_node(expected_stderr) + self.nodes[i].stop_node(expected_stderr, wait=wait) self.nodes[i].wait_until_stopped() - def stop_nodes(self): + def stop_nodes(self, wait=0): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes - node.stop_node() + node.stop_node(wait=wait) for node in self.nodes: # Wait for nodes to stop @@ -374,21 +401,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): sync_blocks(group) sync_mempools(group) - def enable_mocktime(self): - """Enable mocktime for the script. - - mocktime may be needed for scripts that use the cached version of the - blockchain. If the cached version of the blockchain is used without - mocktime then the mempools will not sync due to IBD. - - For backward compatibility of the python scripts with previous - versions of the cache, this helper function sets mocktime to Jan 1, - 2014 + (201 * 10 * 60)""" - self.mocktime = 1388534400 + (201 * 10 * 60) - - def disable_mocktime(self): - self.mocktime = 0 - # Private helper methods. These should not be accessed by the subclass test scripts. def _start_logging(self): @@ -446,7 +458,18 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): args = [self.options.bitcoind, "-datadir=" + datadir, '-disablewallet'] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) - self.nodes.append(TestNode(i, get_datadir_path(self.options.cachedir, i), extra_conf=["bind=127.0.0.1"], extra_args=[], rpchost=None, timewait=self.rpc_timewait, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=None)) + self.nodes.append(TestNode( + i, + get_datadir_path(self.options.cachedir, i), + extra_conf=["bind=127.0.0.1"], + extra_args=[], + rpchost=None, + timewait=self.rpc_timeout, + bitcoind=self.options.bitcoind, + bitcoin_cli=self.options.bitcoincli, + mocktime=self.mocktime, + coverage_dir=None, + )) self.nodes[i].args = args self.start_node(i) @@ -454,6 +477,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: node.wait_for_rpc_connection() + # For backward compatibility of the python scripts with previous + # versions of the cache, set mocktime to Jan 1, + # 2014 + (201 * 10 * 60)""" + self.mocktime = 1388534400 + (201 * 10 * 60) + # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. # Note: To preserve compatibility with older versions of @@ -461,7 +489,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past - self.enable_mocktime() block_time = self.mocktime - (201 * 10 * 60) for i in range(2): for peer in range(4): @@ -475,7 +502,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] - self.disable_mocktime() + self.mocktime = 0 def cache_path(n, *paths): return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c05988c661..999ea68254 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -18,6 +18,8 @@ import tempfile import time import urllib.parse import collections +import shlex +import sys from .authproxy import JSONRPCException from .util import ( @@ -59,7 +61,13 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False): + def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + """ + Kwargs: + start_perf (bool): If True, begin profiling the node with `perf` as soon as + the node starts. + """ + self.index = i self.datadir = datadir self.stdout_dir = os.path.join(self.datadir, "stdout") @@ -68,7 +76,7 @@ class TestNode(): self.rpc_timeout = timewait self.binary = bitcoind self.coverage_dir = coverage_dir - if extra_conf != None: + if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. # For those callers that need more flexibility, they can just set the args property directly. @@ -87,6 +95,7 @@ class TestNode(): self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli + self.start_perf = start_perf self.running = False self.process = None @@ -95,6 +104,8 @@ class TestNode(): self.url = None self.log = logging.getLogger('TestFramework.node%d' % i) self.cleanup_on_exit = True # Whether to kill the node when this object goes away + # Cache perf subprocesses here by their data output filename. + self.perf_subprocesses = {} self.p2ps = [] @@ -115,6 +126,25 @@ class TestNode(): ] return PRIV_KEYS[self.index] + def get_mem_rss_kilobytes(self): + """Get the memory usage (RSS) per `ps`. + + Returns None if `ps` is unavailable. + """ + assert self.running + + try: + return int(subprocess.check_output( + ["ps", "h", "-o", "rss", "{}".format(self.process.pid)], + stderr=subprocess.DEVNULL).split()[-1]) + + # Avoid failing on platforms where ps isn't installed. + # + # We could later use something like `psutils` to work across platforms. + except (FileNotFoundError, subprocess.SubprocessError): + self.log.exception("Unable to get memory usage") + return None + def _node_msg(self, msg: str) -> str: """Return a modified msg that identifies this node by its index as a debugging aid.""" return "[node %d] %s" % (self.index, msg) @@ -167,6 +197,9 @@ class TestNode(): self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") + if self.start_perf: + self._start_perf() + def wait_for_rpc_connection(self): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second @@ -176,12 +209,15 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) - self.rpc.getblockcount() + rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up + self.log.debug("RPC successfully started") + if self.use_cli: + return + self.rpc = rpc self.rpc_connected = True self.url = self.rpc.url - self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? @@ -197,6 +233,10 @@ class TestNode(): time.sleep(1.0 / poll_per_s) self._raise_assertion_error("Unable to connect to bitcoind") + def generate(self, nblocks, maxtries=1000000): + self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") + return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries) + def get_wallet_rpc(self, wallet_name): if self.use_cli: return self.cli("-rpcwallet={}".format(wallet_name)) @@ -205,16 +245,20 @@ class TestNode(): wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name)) return self.rpc / wallet_path - def stop_node(self, expected_stderr=''): + def stop_node(self, expected_stderr='', wait=0): """Stop the node.""" if not self.running: return self.log.debug("Stopping node") try: - self.stop() + self.stop(wait=wait) except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") + # If there are any running perf processes, stop them. + for profile_name in tuple(self.perf_subprocesses.keys()): + self._stop_perf(profile_name) + # Check that stderr is as expected self.stderr.seek(0) stderr = self.stderr.read().decode('utf-8').strip() @@ -267,6 +311,111 @@ class TestNode(): if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: self._raise_assertion_error('Expected message "{}" does not partially match log:\n\n{}\n\n'.format(expected_msg, print_log)) + @contextlib.contextmanager + def assert_memory_usage_stable(self, *, increase_allowed=0.03): + """Context manager that allows the user to assert that a node's memory usage (RSS) + hasn't increased beyond some threshold percentage. + + Args: + increase_allowed (float): the fractional increase in memory allowed until failure; + e.g. `0.12` for up to 12% increase allowed. + """ + before_memory_usage = self.get_mem_rss_kilobytes() + + yield + + after_memory_usage = self.get_mem_rss_kilobytes() + + if not (before_memory_usage and after_memory_usage): + self.log.warning("Unable to detect memory usage (RSS) - skipping memory check.") + return + + perc_increase_memory_usage = (after_memory_usage / before_memory_usage) - 1 + + if perc_increase_memory_usage > increase_allowed: + self._raise_assertion_error( + "Memory usage increased over threshold of {:.3f}% from {} to {} ({:.3f}%)".format( + increase_allowed * 100, before_memory_usage, after_memory_usage, + perc_increase_memory_usage * 100)) + + @contextlib.contextmanager + def profile_with_perf(self, profile_name): + """ + Context manager that allows easy profiling of node activity using `perf`. + + See `test/functional/README.md` for details on perf usage. + + Args: + profile_name (str): This string will be appended to the + profile data filename generated by perf. + """ + subp = self._start_perf(profile_name) + + yield + + if subp: + self._stop_perf(profile_name) + + def _start_perf(self, profile_name=None): + """Start a perf process to profile this node. + + Returns the subprocess running perf.""" + subp = None + + def test_success(cmd): + return subprocess.call( + # shell=True required for pipe use below + cmd, shell=True, + stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0 + + if not sys.platform.startswith('linux'): + self.log.warning("Can't profile with perf; only availabe on Linux platforms") + return None + + if not test_success('which perf'): + self.log.warning("Can't profile with perf; must install perf-tools") + return None + + if not test_success('readelf -S {} | grep .debug_str'.format(shlex.quote(self.binary))): + self.log.warning( + "perf output won't be very useful without debug symbols compiled into bitcoind") + + output_path = tempfile.NamedTemporaryFile( + dir=self.datadir, + prefix="{}.perf.data.".format(profile_name or 'test'), + delete=False, + ).name + + cmd = [ + 'perf', 'record', + '-g', # Record the callgraph. + '--call-graph', 'dwarf', # Compatibility for gcc's --fomit-frame-pointer. + '-F', '101', # Sampling frequency in Hz. + '-p', str(self.process.pid), + '-o', output_path, + ] + subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.perf_subprocesses[profile_name] = subp + + return subp + + def _stop_perf(self, profile_name): + """Stop (and pop) a perf subprocess.""" + subp = self.perf_subprocesses.pop(profile_name) + output_path = subp.args[subp.args.index('-o') + 1] + + subp.terminate() + subp.wait(timeout=10) + + stderr = subp.stderr.read().decode() + if 'Consider tweaking /proc/sys/kernel/perf_event_paranoid' in stderr: + self.log.warning( + "perf couldn't collect data! Try " + "'sudo sysctl -w kernel.perf_event_paranoid=-1'") + else: + report_cmd = "perf report -i {}".format(output_path) + self.log.info("See perf output by running '{}'".format(report_cmd)) + def assert_start_raises_init_error(self, extra_args=None, expected_msg=None, match=ErrorMatch.FULL_TEXT, *args, **kwargs): """Attempt to start the node and expect it to raise an error. @@ -352,6 +501,14 @@ class TestNodeCLIAttr: def get_request(self, *args, **kwargs): return lambda: self(*args, **kwargs) +def arg_to_cli(arg): + if isinstance(arg, bool): + return str(arg).lower() + elif isinstance(arg, dict) or isinstance(arg, list): + return json.dumps(arg) + else: + return str(arg) + class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" @@ -383,8 +540,8 @@ class TestNodeCLI(): def send_cli(self, command=None, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" - pos_args = [str(arg).lower() if type(arg) is bool else str(arg) for arg in args] - named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()] + pos_args = [arg_to_cli(arg) for arg in args] + named_args = [str(key) + "=" + arg_to_cli(value) for (key, value) in kwargs.items()] assert not (pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call" p_args = [self.binary, "-datadir=" + self.datadir] + self.options if named_args: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index b355816d8b..d0a78d8dfd 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -326,7 +326,7 @@ def get_auth_cookie(datadir): if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") - if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): + if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")) and os.access(os.path.join(datadir, "regtest", ".cookie"), os.R_OK): with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py new file mode 100755 index 0000000000..c0dfa4c3f0 --- /dev/null +++ b/test/functional/test_framework/wallet_util.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Useful util functions for testing the wallet""" +from collections import namedtuple + +from test_framework.address import ( + key_to_p2pkh, + key_to_p2sh_p2wpkh, + key_to_p2wpkh, + script_to_p2sh, + script_to_p2sh_p2wsh, + script_to_p2wsh, +) +from test_framework.script import ( + CScript, + OP_0, + OP_2, + OP_3, + OP_CHECKMULTISIG, + OP_CHECKSIG, + OP_DUP, + OP_EQUAL, + OP_EQUALVERIFY, + OP_HASH160, + hash160, + sha256, +) +from test_framework.util import hex_str_to_bytes + +Key = namedtuple('Key', ['privkey', + 'pubkey', + 'p2pkh_script', + 'p2pkh_addr', + 'p2wpkh_script', + 'p2wpkh_addr', + 'p2sh_p2wpkh_script', + 'p2sh_p2wpkh_redeem_script', + 'p2sh_p2wpkh_addr']) + +Multisig = namedtuple('Multisig', ['privkeys', + 'pubkeys', + 'p2sh_script', + 'p2sh_addr', + 'redeem_script', + 'p2wsh_script', + 'p2wsh_addr', + 'p2sh_p2wsh_script', + 'p2sh_p2wsh_addr']) + +def get_key(node): + """Generate a fresh key on node + + Returns a named tuple of privkey, pubkey and all address and scripts.""" + addr = node.getnewaddress() + pubkey = node.getaddressinfo(addr)['pubkey'] + pkh = hash160(hex_str_to_bytes(pubkey)) + return Key(privkey=node.dumpprivkey(addr), + pubkey=pubkey, + p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_addr=key_to_p2pkh(pubkey), + p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_addr=key_to_p2wpkh(pubkey), + p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), + p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) + +def get_multisig(node): + """Generate a fresh 2-of-3 multisig on node + + Returns a named tuple of privkeys, pubkeys and all address and scripts.""" + addrs = [] + pubkeys = [] + for _ in range(3): + addr = node.getaddressinfo(node.getnewaddress()) + addrs.append(addr['address']) + pubkeys.append(addr['pubkey']) + script_code = CScript([OP_2] + [hex_str_to_bytes(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) + witness_script = CScript([OP_0, sha256(script_code)]) + return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs], + pubkeys=pubkeys, + p2sh_script=CScript([OP_HASH160, hash160(script_code), OP_EQUAL]).hex(), + p2sh_addr=script_to_p2sh(script_code), + redeem_script=script_code.hex(), + p2wsh_script=witness_script.hex(), + p2wsh_addr=script_to_p2wsh(script_code), + p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(), + p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code)) + +def test_address(node, address, **kwargs): + """Get address info for `address` and test whether the returned values are as expected.""" + addr_info = node.getaddressinfo(address) + for key, value in kwargs.items(): + if value is None: + if key in addr_info.keys(): + raise AssertionError("key {} unexpectedly returned in getaddressinfo.".format(key)) + elif addr_info[key] != value: + raise AssertionError("key {} value {} did not match expected value {}".format(key, addr_info[key], value)) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 3f990924ec..d8f92e2601 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -7,8 +7,6 @@ This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. -Functional tests are disabled on Windows by default. Use --force to run them anyway. - For a description of arguments recognized by test scripts, see `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -68,9 +66,6 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 -# 20 minutes represented in seconds -TRAVIS_TIMEOUT_DURATION = 20 * 60 - BASE_SCRIPTS = [ # Scripts that are run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -111,6 +106,7 @@ BASE_SCRIPTS = [ 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', + 'tool_wallet.py', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', @@ -120,9 +116,10 @@ BASE_SCRIPTS = [ 'mempool_persist.py', 'wallet_multiwallet.py', 'wallet_multiwallet.py --usecli', - 'wallet_disableprivatekeys.py', - 'wallet_disableprivatekeys.py --usecli', + 'wallet_createwallet.py', + 'wallet_createwallet.py --usecli', 'interface_http.py', + 'interface_rpc.py', 'rpc_psbt.py', 'rpc_users.py', 'feature_proxy.py', @@ -139,6 +136,7 @@ BASE_SCRIPTS = [ 'mining_prioritisetransaction.py', 'p2p_invalid_locator.py', 'p2p_invalid_block.py', + 'p2p_invalid_messages.py', 'p2p_invalid_tx.py', 'feature_assumevalid.py', 'example_test.py', @@ -152,10 +150,13 @@ BASE_SCRIPTS = [ 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py', + 'p2p_leak_tx.py', 'rpc_signmessage.py', + 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', 'wallet_import_rescan.py', + 'wallet_import_with_label.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', @@ -172,11 +173,15 @@ BASE_SCRIPTS = [ 'wallet_fallbackfee.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', + 'wallet_create_tx.py', 'p2p_fingerprint.py', 'feature_uacomment.py', + 'wallet_coinbase_category.py', 'feature_filelock.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', + 'rpc_deriveaddresses.py', + 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', @@ -184,6 +189,7 @@ BASE_SCRIPTS = [ 'feature_config_args.py', 'rpc_help.py', 'feature_help.py', + 'feature_shutdown.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -213,15 +219,15 @@ def main(): epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--combinedlogslen', '-c', type=int, default=0, help='print a combined log (of length n lines) from all test nodes and test framework to the console on failure.') + parser.add_argument('--combinedlogslen', '-c', type=int, default=0, metavar='n', help='On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes.') parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') - parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') - parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') + parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') args, unknown_args = parser.parse_known_args() @@ -254,12 +260,6 @@ def main(): enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - if config["environment"]["EXEEXT"] == ".exe" and not args.force: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Tests currently disabled on Windows by default. Use --force option to enable") - sys.exit(0) - if not enable_bitcoind: print("No functional tests to run.") print("Rerun ./configure with --with-daemon and then make") @@ -270,7 +270,7 @@ def main(): if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. - tests = [re.sub("\.py$", "", test) + ".py" for test in tests] + tests = [test + ".py" if ".py" not in test else test for test in tests] for test in tests: if test in ALL_SCRIPTS: test_list.append(test) @@ -305,25 +305,26 @@ def main(): subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h']) sys.exit(0) - check_script_list(config["environment"]["SRCDIR"]) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) check_script_prefixes() if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests( - test_list, - config["environment"]["SRCDIR"], - config["environment"]["BUILDDIR"], - tmpdir, + test_list=test_list, + src_dir=config["environment"]["SRCDIR"], + build_dir=config["environment"]["BUILDDIR"], + tmpdir=tmpdir, jobs=args.jobs, enable_coverage=args.coverage, args=passon_args, combined_logs_len=args.combinedlogslen, failfast=args.failfast, + runs_ci=args.ci, ) -def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, runs_ci): args = args or [] # Warn if bitcoind is already running (unix only) @@ -358,22 +359,29 @@ def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=Fal raise #Run Tests - job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) + job_queue = TestHandler( + num_tests_parallel=jobs, + tests_dir=tests_dir, + tmpdir=tmpdir, + test_list=test_list, + flags=flags, + timeout_duration=40 * 60 if runs_ci else float('inf'), # in seconds + ) start_time = time.time() test_results = [] max_len_name = len(max(test_list, key=len)) - - for _ in range(len(test_list)): + test_count = len(test_list) + for i in range(test_count): test_result, testdir, stdout, stderr = job_queue.get_next() test_results.append(test_result) - + done_str = "{}/{} - {}{}{}".format(i + 1, test_count, BOLD[1], test_result.name, BOLD[0]) if test_result.status == "Passed": - logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) elif test_result.status == "Skipped": - logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0])) + logging.debug("%s skipped" % (done_str)) else: - print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') if combined_logs_len and os.path.isdir(testdir): @@ -439,11 +447,12 @@ class TestHandler: Trigger the test scripts passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): - assert(num_tests_parallel >= 1) + def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, timeout_duration): + assert num_tests_parallel >= 1 self.num_jobs = num_tests_parallel self.tests_dir = tests_dir self.tmpdir = tmpdir + self.timeout_duration = timeout_duration self.test_list = test_list self.flags = flags self.num_running = 0 @@ -472,12 +481,13 @@ class TestHandler: log_stderr)) if not self.jobs: raise IndexError('pop from empty list') + dot_count = 0 while True: # Return first proc that finishes time.sleep(.5) for job in self.jobs: (name, start_time, proc, testdir, log_out, log_err) = job - if os.getenv('TRAVIS') == 'true' and int(time.time() - start_time) > TRAVIS_TIMEOUT_DURATION: + if int(time.time() - start_time) > self.timeout_duration: # In travis, timeout individual tests (to stop tests hanging and not providing useful output). proc.send_signal(signal.SIGINT) if proc.poll() is not None: @@ -492,9 +502,12 @@ class TestHandler: status = "Failed" self.num_running -= 1 self.jobs.remove(job) - + clearline = '\r' + (' ' * dot_count) + '\r' + print(clearline, end='', flush=True) + dot_count = 0 return TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr print('.', end='', flush=True) + dot_count += 1 def kill_and_join(self): """Send SIGKILL to all jobs and block until all have ended.""" @@ -543,7 +556,7 @@ class TestResult(): def check_script_prefixes(): """Check that test scripts start with one of the allowed name prefixes.""" - good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet)_") + good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] if bad_script_names: @@ -552,7 +565,7 @@ def check_script_prefixes(): raise AssertionError("Some tests are not following naming convention!") -def check_script_list(src_dir): +def check_script_list(*, src_dir, fail_on_warn): """Check scripts directory. Check that there are no scripts in the functional tests directory which are @@ -562,10 +575,11 @@ def check_script_list(src_dir): missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) - if os.getenv('TRAVIS') == 'true': + if fail_on_warn: # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) + class RPCCoverage(): """ Coverage reporting utilities for test_runner. @@ -621,7 +635,7 @@ class RPCCoverage(): with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file: all_cmds.update([line.strip() for line in coverage_ref_file.readlines()]) - for root, dirs, files in os.walk(self.dir): + for root, _, files in os.walk(self.dir): for filename in files: if filename.startswith(coverage_file_prefix): coverage_filenames.add(os.path.join(root, filename)) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py new file mode 100755 index 0000000000..fbcf21e729 --- /dev/null +++ b/test/functional/tool_wallet.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoin-wallet.""" +import subprocess +import textwrap + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class ToolWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def bitcoin_wallet_process(self, *args): + binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] + args = ['-datadir={}'.format(self.nodes[0].datadir), '-regtest'] + list(args) + return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + def assert_raises_tool_error(self, error, *args): + p = self.bitcoin_wallet_process(*args) + stdout, stderr = p.communicate() + assert_equal(p.poll(), 1) + assert_equal(stdout, '') + assert_equal(stderr.strip(), error) + + def assert_tool_output(self, output, *args): + p = self.bitcoin_wallet_process(*args) + stdout, stderr = p.communicate() + assert_equal(p.poll(), 0) + assert_equal(stderr, '') + assert_equal(stdout, output) + + def run_test(self): + + self.assert_raises_tool_error('Invalid command: foo', 'foo') + # `bitcoin-wallet help` is an error. Use `bitcoin-wallet -help` + self.assert_raises_tool_error('Invalid command: help', 'help') + self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info') + self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') + + # stop the node to close the wallet to call info command + self.stop_node(0) + + out = textwrap.dedent('''\ + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2 + Transactions: 0 + Address Book: 3 + ''') + self.assert_tool_output(out, '-wallet=wallet.dat', 'info') + + # mutate the wallet to check the info command output changes accordingly + self.start_node(0) + self.nodes[0].generate(1) + self.stop_node(0) + + out = textwrap.dedent('''\ + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2 + Transactions: 1 + Address Book: 3 + ''') + self.assert_tool_output(out, '-wallet=wallet.dat', 'info') + + out = textwrap.dedent('''\ + Topping up keypool... + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2000 + Transactions: 0 + Address Book: 0 + ''') + self.assert_tool_output(out, '-wallet=foo', 'create') + + self.start_node(0, ['-wallet=foo']) + out = self.nodes[0].getwalletinfo() + self.stop_node(0) + + assert_equal(0, out['txcount']) + assert_equal(1000, out['keypoolsize']) + assert_equal(1000, out['keypoolsize_hd_internal']) + assert_equal(True, 'hdseedid' in out) + +if __name__ == '__main__': + ToolWalletTest().main() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index e5ac2c8bd4..0c3c247694 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. @@ -13,12 +13,21 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, + sync_blocks, + sync_mempools, +) + class AbandonConflictTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.extra_args = [["-minrelaytxfee=0.00001"], []] + # TODO: remove -txindex. Currently required for getrawtransaction call. + self.extra_args = [["-minrelaytxfee=0.00001", "-txindex"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -40,21 +49,21 @@ class AbandonConflictTest(BitcoinTestFramework): sync_blocks(self.nodes) newbalance = self.nodes[0].getbalance() - assert(balance - newbalance < Decimal("0.001")) #no more than fees lost + assert balance - newbalance < Decimal("0.001") #no more than fees lost balance = newbalance # Disconnect nodes so node0's transactions don't get into node1's mempool disconnect_nodes(self.nodes[0], 1) # Identify the 10btc outputs - nA = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txA, 1)["vout"]) if vout["value"] == Decimal("10")) - nB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txB, 1)["vout"]) if vout["value"] == Decimal("10")) - nC = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txC, 1)["vout"]) if vout["value"] == Decimal("10")) + nA = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txA)["details"] if tx_out["amount"] == Decimal("10")) + nB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txB)["details"] if tx_out["amount"] == Decimal("10")) + nC = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txC)["details"] if tx_out["amount"] == Decimal("10")) - inputs =[] + inputs = [] # spend 10btc outputs from txA and txB - inputs.append({"txid":txA, "vout":nA}) - inputs.append({"txid":txB, "vout":nB}) + inputs.append({"txid": txA, "vout": nA}) + inputs.append({"txid": txB, "vout": nB}) outputs = {} outputs[self.nodes[0].getnewaddress()] = Decimal("14.99998") @@ -63,12 +72,12 @@ class AbandonConflictTest(BitcoinTestFramework): txAB1 = self.nodes[0].sendrawtransaction(signed["hex"]) # Identify the 14.99998btc output - nAB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txAB1, 1)["vout"]) if vout["value"] == Decimal("14.99998")) + nAB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txAB1)["details"] if tx_out["amount"] == Decimal("14.99998")) #Create a child tx spending AB1 and C inputs = [] - inputs.append({"txid":txAB1, "vout":nAB}) - inputs.append({"txid":txC, "vout":nC}) + inputs.append({"txid": txAB1, "vout": nAB}) + inputs.append({"txid": txC, "vout": nC}) outputs = {} outputs[self.nodes[0].getnewaddress()] = Decimal("24.9996") signed2 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) @@ -76,8 +85,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Create a child tx spending ABC2 signed3_change = Decimal("24.999") - inputs = [ {"txid":txABC2, "vout":0} ] - outputs = { self.nodes[0].getnewaddress(): signed3_change } + inputs = [{"txid": txABC2, "vout": 0}] + outputs = {self.nodes[0].getnewaddress(): signed3_change} signed3 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) # note tx is never directly referenced, only abandoned as a child of the above self.nodes[0].sendrawtransaction(signed3["hex"]) @@ -105,7 +114,7 @@ class AbandonConflictTest(BitcoinTestFramework): unconfbalance = self.nodes[0].getunconfirmedbalance() + self.nodes[0].getbalance() assert_equal(unconfbalance, newbalance) # Also shouldn't show up in listunspent - assert(not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)]) + assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)] balance = newbalance # Abandon original transaction and verify inputs are available again @@ -145,8 +154,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Create a double spend of AB1 by spending again from only A's 10 output # Mine double spend from node 1 - inputs =[] - inputs.append({"txid":txA, "vout":nA}) + inputs = [] + inputs.append({"txid": txA, "vout": nA}) outputs = {} outputs[self.nodes[1].getnewaddress()] = Decimal("9.9999") tx = self.nodes[0].createrawtransaction(inputs, outputs) @@ -172,5 +181,6 @@ class AbandonConflictTest(BitcoinTestFramework): self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") self.log.info(str(balance) + " -> " + str(newbalance) + " ?") + if __name__ == '__main__': AbandonConflictTest().main() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 0f75045c9d..bafa556aad 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -99,6 +99,8 @@ class AddressTypeTest(BitcoinTestFramework): """Run sanity checks on an address.""" info = self.nodes[node].getaddressinfo(address) assert(self.nodes[node].validateaddress(address)['isvalid']) + assert_equal(info.get('solvable'), True) + if not multisig and typ == 'legacy': # P2PKH assert(not info['isscript']) @@ -146,6 +148,47 @@ class AddressTypeTest(BitcoinTestFramework): # Unknown type assert(False) + def test_desc(self, node, address, multisig, typ, utxo): + """Run sanity checks on a descriptor reported by getaddressinfo.""" + info = self.nodes[node].getaddressinfo(address) + assert('desc' in info) + assert_equal(info['desc'], utxo['desc']) + assert(self.nodes[node].validateaddress(address)['isvalid']) + + # Use a ridiculously roundabout way to find the key origin info through + # the PSBT logic. However, this does test consistency between the PSBT reported + # fingerprints/paths and the descriptor logic. + psbt = self.nodes[node].createpsbt([{'txid':utxo['txid'], 'vout':utxo['vout']}],[{address:0.00010000}]) + psbt = self.nodes[node].walletprocesspsbt(psbt, False, "ALL", True) + decode = self.nodes[node].decodepsbt(psbt['psbt']) + key_descs = {} + for deriv in decode['inputs'][0]['bip32_derivs']: + assert_equal(len(deriv['master_fingerprint']), 8) + assert_equal(deriv['path'][0], 'm') + key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] + + if not multisig and typ == 'legacy': + # P2PKH + assert_equal(info['desc'], "pkh(%s)" % key_descs[info['pubkey']]) + elif not multisig and typ == 'p2sh-segwit': + # P2SH-P2WPKH + assert_equal(info['desc'], "sh(wpkh(%s))" % key_descs[info['pubkey']]) + elif not multisig and typ == 'bech32': + # P2WPKH + assert_equal(info['desc'], "wpkh(%s)" % key_descs[info['pubkey']]) + elif typ == 'legacy': + # P2SH-multisig + assert_equal(info['desc'], "sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + elif typ == 'p2sh-segwit': + # P2SH-P2WSH-multisig + assert_equal(info['desc'], "sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]])) + elif typ == 'bech32': + # P2WSH-multisig + assert_equal(info['desc'], "wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + else: + # Unknown type + assert(False) + def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) raw_tx = self.nodes[node_sender].getrawtransaction(txid) @@ -198,6 +241,7 @@ class AddressTypeTest(BitcoinTestFramework): self.log.debug("Old balances are {}".format(old_balances)) to_send = (old_balances[from_node] / 101).quantize(Decimal("0.00000001")) sends = {} + addresses = {} self.log.debug("Prepare sends") for n, to_node in enumerate(range(from_node, from_node + 4)): @@ -228,6 +272,7 @@ class AddressTypeTest(BitcoinTestFramework): # Output entry sends[address] = to_send * 10 * (1 + n) + addresses[to_node] = (address, typ) self.log.debug("Sending: {}".format(sends)) self.nodes[from_node].sendmany("", sends) @@ -244,6 +289,17 @@ class AddressTypeTest(BitcoinTestFramework): self.nodes[5].generate(1) sync_blocks(self.nodes) + # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor + for n, to_node in enumerate(range(from_node, from_node + 4)): + to_node %= 4 + found = False + for utxo in self.nodes[to_node].listunspent(): + if utxo['address'] == addresses[to_node][0]: + found = True + self.test_desc(to_node, addresses[to_node][0], multisig, addresses[to_node][1], utxo) + break + assert found + new_balances = self.get_balances() self.log.debug("Check new balances: {}".format(new_balances)) # We don't know what fee was set, so we can only check bounds on the balance of the sending node diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 32ec385fa1..dd3750203a 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -48,7 +48,7 @@ class WalletBackupTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.setup_nodes() connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py new file mode 100755 index 0000000000..05c97e0340 --- /dev/null +++ b/test/functional/wallet_balance.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the wallet balance RPC methods.""" +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +RANDOM_COINBASE_ADDRESS = 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ' + +def create_transactions(node, address, amt, fees): + # Create and sign raw transactions from node to address for amt. + # Creates a transaction for each fee and returns an array + # of the raw transactions. + utxos = node.listunspent(0) + + # Create transactions + inputs = [] + ins_total = 0 + for utxo in utxos: + inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]}) + ins_total += utxo['amount'] + if ins_total > amt: + break + + txs = [] + for fee in fees: + outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee} + raw_tx = node.createrawtransaction(inputs, outputs, 0, True) + raw_tx = node.signrawtransactionwithwallet(raw_tx) + txs.append(raw_tx) + + return txs + +class WalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + # Check that nodes don't own any UTXOs + assert_equal(len(self.nodes[0].listunspent()), 0) + assert_equal(len(self.nodes[1].listunspent()), 0) + + self.log.info("Mining one block for each node") + + self.nodes[0].generate(1) + self.sync_all() + self.nodes[1].generate(1) + self.nodes[1].generatetoaddress(100, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + assert_equal(self.nodes[0].getbalance(), 50) + assert_equal(self.nodes[1].getbalance(), 50) + + self.log.info("Test getbalance with different arguments") + assert_equal(self.nodes[0].getbalance("*"), 50) + assert_equal(self.nodes[0].getbalance("*", 1), 50) + assert_equal(self.nodes[0].getbalance("*", 1, True), 50) + assert_equal(self.nodes[0].getbalance(minconf=1), 50) + + # Send 40 BTC from 0 to 1 and 60 BTC from 1 to 0. + txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')]) + self.nodes[0].sendrawtransaction(txs[0]['hex']) + self.nodes[1].sendrawtransaction(txs[0]['hex']) # sending on both nodes is faster than waiting for propagation + + self.sync_all() + txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), 60, [Decimal('0.01'), Decimal('0.02')]) + self.nodes[1].sendrawtransaction(txs[0]['hex']) + self.nodes[0].sendrawtransaction(txs[0]['hex']) # sending on both nodes is faster than waiting for propagation + self.sync_all() + + # First argument of getbalance must be set to "*" + assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "") + + self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + + # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions + assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('29.99')) # change from node 1's send + # Same with minconf=0 + assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99')) + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) + assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) + # getunconfirmedbalance + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + + # Node 1 bumps the transaction fee and resends + self.nodes[1].sendrawtransaction(txs[1]['hex']) + self.sync_all() + + self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) # output of node 1's send + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) + + self.nodes[1].generatetoaddress(1, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + # balances are correct after the transactions are confirmed + assert_equal(self.nodes[0].getbalance(), Decimal('69.99')) # node 1's send plus change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('29.98')) # change from node 0's send + + # Send total balance away from node 1 + txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) + self.nodes[1].sendrawtransaction(txs[0]['hex']) + self.nodes[1].generatetoaddress(2, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + # getbalance with minconf=3 should still show the old balance + assert_equal(self.nodes[1].getbalance(minconf=3), Decimal('0')) + + # getbalance with minconf=2 will show the new balance. + assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0')) + +if __name__ == '__main__': + WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 8bbff7f7ef..673eb718e6 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" @@ -11,6 +11,7 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_fee_amount, + assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, sync_blocks, @@ -18,19 +19,21 @@ from test_framework.util import ( wait_until, ) + class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True + # TODO: remove -txindex. Currently required for getrawtransaction call. + self.extra_args = [[], [], ["-txindex"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def setup_network(self): - self.add_nodes(4) - self.start_node(0) - self.start_node(1) - self.start_node(2) + self.setup_nodes() + # Only need nodes 0-2 running at start of test + self.stop_node(3) connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) @@ -67,15 +70,6 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 50) assert_equal(self.nodes[2].getbalance(), 0) - # Check getbalance with different arguments - assert_equal(self.nodes[0].getbalance("*"), 50) - assert_equal(self.nodes[0].getbalance("*", 1), 50) - assert_equal(self.nodes[0].getbalance("*", 1, True), 50) - assert_equal(self.nodes[0].getbalance(minconf=1), 50) - - # first argument of getbalance must be excluded or set to "*" - assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[0].getbalance, "") - # Check that only first and second nodes have UTXOs utxos = self.nodes[0].listunspent() assert_equal(len(utxos), 1) @@ -92,13 +86,13 @@ class WalletTest(BitcoinTestFramework): assert_equal(txout['value'], 50) # Send 21 BTC from 0 to 2 using sendtoaddress call. - # Locked memory should use at least 32 bytes to sign each transaction + # Locked memory should increase to sign transactions self.log.info("test getmemoryinfo") memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) memory_after = self.nodes[0].getmemoryinfo() - assert(memory_before['locked']['used'] + 64 <= memory_after['locked']['used']) + assert_greater_than(memory_after['locked']['used'], memory_before['locked']['used']) self.log.info("test gettxout (second part)") # utxo spent in mempool should be visible if you exclude mempool @@ -203,7 +197,7 @@ class WalletTest(BitcoinTestFramework): txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) self.nodes[2].generate(1) self.sync_all([self.nodes[0:3]]) - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount @@ -212,14 +206,14 @@ class WalletTest(BitcoinTestFramework): self.sync_all([self.nodes[0:3]]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) # Sendmany 10 BTC txid = self.nodes[2].sendmany('', {address: 10}, 0, "", []) self.nodes[2].generate(1) self.sync_all([self.nodes[0:3]]) node_0_bal += Decimal('10') - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), node_0_bal) # Sendmany 10 BTC with subtract fee from amount @@ -228,7 +222,7 @@ class WalletTest(BitcoinTestFramework): self.sync_all([self.nodes[0:3]]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) # Test ResendWalletTransactions: # Create a couple of transactions, then start up a fourth @@ -246,11 +240,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(set(relayed), {txid1, txid2}) sync_mempools(self.nodes) - assert(txid1 in self.nodes[3].getrawmempool()) - - # Exercise balance rpcs - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], 1) - assert_equal(self.nodes[0].getunconfirmedbalance(), 1) + assert txid1 in self.nodes[3].getrawmempool() # check if we can list zero value tx as available coins # 1. create raw_tx @@ -277,7 +267,7 @@ class WalletTest(BitcoinTestFramework): if uTx['txid'] == zero_value_txid: found = True assert_equal(uTx['amount'], Decimal('0')) - assert(found) + assert found # do some -walletbroadcast tests self.stop_nodes() @@ -354,7 +344,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[1].importaddress(address_to_import) # 3. Validate that the imported address is watch-only on node1 - assert(self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"]) + assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] # 4. Check that the unspents after import are not spendable assert_array_result(self.nodes[1].listunspent(), @@ -396,7 +386,7 @@ class WalletTest(BitcoinTestFramework): addr = self.nodes[0].getnewaddress() self.nodes[0].setlabel(addr, label) assert_equal(self.nodes[0].getaddressinfo(addr)['label'], label) - assert(label in self.nodes[0].listlabels()) + assert label in self.nodes[0].listlabels() self.nodes[0].rpc.ensure_ascii = True # restore to default # maintenance tests @@ -455,8 +445,8 @@ class WalletTest(BitcoinTestFramework): # Without walletrejectlongchains, we will still generate a txid # The tx will be stored in the wallet but not accepted to the mempool extra_txid = self.nodes[0].sendtoaddress(sending_addr, Decimal('0.0001')) - assert(extra_txid not in self.nodes[0].getrawmempool()) - assert(extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()]) + assert extra_txid not in self.nodes[0].getrawmempool() + assert extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()] self.nodes[0].abandontransaction(extra_txid) total_txs = len(self.nodes[0].listtransactions("*", 99999)) @@ -479,7 +469,7 @@ class WalletTest(BitcoinTestFramework): # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) - # Test getaddressinfo. Note that these addresses are taken from disablewallet.py + # Test getaddressinfo on external address. Note that these addresses are taken from disablewallet.py assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") address_info = self.nodes[0].getaddressinfo("mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info['address'], "mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") @@ -487,6 +477,23 @@ class WalletTest(BitcoinTestFramework): assert not address_info["ismine"] assert not address_info["iswatchonly"] assert not address_info["isscript"] + assert not address_info["ischange"] + + # Test getaddressinfo 'ischange' field on change address. + self.nodes[0].generate(1) + destination = self.nodes[1].getnewaddress() + txid = self.nodes[0].sendtoaddress(destination, 0.123) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + output_addresses = [vout['scriptPubKey']['addresses'][0] for vout in tx["vout"]] + assert len(output_addresses) > 1 + for address in output_addresses: + ischange = self.nodes[0].getaddressinfo(address)['ischange'] + assert_equal(ischange, address != destination) + if ischange: + change = address + self.nodes[0].setlabel(change, 'foobar') + assert_equal(self.nodes[0].getaddressinfo(change)['ischange'], False) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 67ee00871d..7d3d9b61e2 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -13,26 +13,22 @@ can be disabled or reordered if needed for debugging. If new test cases are added in the future, they should try to follow the same convention and not make assumptions about execution order. """ - from decimal import Decimal +import io from test_framework.blocktools import add_witness_commitment, create_block, create_coinbase, send_to_witness from test_framework.messages import BIP125_SEQUENCE_NUMBER, CTransaction from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes, sync_mempools -import io - WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 - class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True self.extra_args = [[ - "-deprecatedrpc=addwitnessaddress", "-walletrbf={}".format(i), "-mintxfee=0.00002", ] for i in range(self.num_nodes)] @@ -107,8 +103,7 @@ def test_segwit_bumpfee_succeeds(rbf_node, dest_address): # which spends it, and make sure bumpfee can be called on it. segwit_in = next(u for u in rbf_node.listunspent() if u["amount"] == Decimal("0.001")) - segwit_out = rbf_node.getaddressinfo(rbf_node.getnewaddress()) - rbf_node.addwitnessaddress(segwit_out["address"]) + segwit_out = rbf_node.getaddressinfo(rbf_node.getnewaddress(address_type='p2sh-segwit')) segwitid = send_to_witness( use_p2wsh=False, node=rbf_node, @@ -157,7 +152,7 @@ def test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address): signedtx = peer_node.signrawtransactionwithwallet(signedtx["hex"]) rbfid = rbf_node.sendrawtransaction(signedtx["hex"]) assert_raises_rpc_error(-4, "Transaction contains inputs that don't belong to this wallet", - rbf_node.bumpfee, rbfid) + rbf_node.bumpfee, rbfid) def test_bumpfee_with_descendant_fails(rbf_node, rbf_node_address, dest_address): @@ -187,11 +182,11 @@ def test_dust_to_fee(rbf_node, dest_address): # (32-byte p2sh-pwpkh output size + 148 p2pkh spend estimate) * 10k(discard_rate) / 1000 = 1800 # P2SH outputs are slightly "over-discarding" due to the IsDust calculation assuming it will # be spent as a P2PKH. - bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000-1800}) + bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 1800}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) - assert_equal(len(full_bumped_tx["vout"]), 1) #change output is eliminated + assert_equal(len(full_bumped_tx["vout"]), 1) # change output is eliminated def test_settxfee(rbf_node, dest_address): @@ -222,7 +217,7 @@ def test_rebumping_not_replaceable(rbf_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) bumped = rbf_node.bumpfee(rbfid, {"totalFee": 10000, "replaceable": False}) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], - {"totalFee": 20000}) + {"totalFee": 20000}) def test_unconfirmed_not_spendable(rbf_node, rbf_node_address): @@ -276,7 +271,7 @@ def test_locked_wallet_fails(rbf_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) rbf_node.walletlock() assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", - rbf_node.bumpfee, rbfid) + rbf_node.bumpfee, rbfid) def spend_one_input(node, dest_address): diff --git a/test/functional/wallet_coinbase_category.py b/test/functional/wallet_coinbase_category.py new file mode 100755 index 0000000000..7aa8b44ebd --- /dev/null +++ b/test/functional/wallet_coinbase_category.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test coinbase transactions return the correct categories. + +Tests listtransactions, listsinceblock, and gettransaction. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_array_result +) + +class CoinbaseCategoryTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def assert_category(self, category, address, txid, skip): + assert_array_result(self.nodes[0].listtransactions(skip=skip), + {"address": address}, + {"category": category}) + assert_array_result(self.nodes[0].listsinceblock()["transactions"], + {"address": address}, + {"category": category}) + assert_array_result(self.nodes[0].gettransaction(txid)["details"], + {"address": address}, + {"category": category}) + + def run_test(self): + # Generate one block to an address + address = self.nodes[0].getnewaddress() + self.nodes[0].generatetoaddress(1, address) + hash = self.nodes[0].getbestblockhash() + txid = self.nodes[0].getblock(hash)["tx"][0] + + # Coinbase transaction is immature after 1 confirmation + self.assert_category("immature", address, txid, 0) + + # Mine another 99 blocks on top + self.nodes[0].generate(99) + # Coinbase transaction is still immature after 100 confirmations + self.assert_category("immature", address, txid, 99) + + # Mine one more block + self.nodes[0].generate(1) + # Coinbase transaction is now matured, so category is "generate" + self.assert_category("generate", address, txid, 100) + + # Orphan block that paid to address + self.nodes[0].invalidateblock(hash) + # Coinbase transaction is now orphaned + self.assert_category("orphan", address, txid, 100) + +if __name__ == '__main__': + CoinbaseCategoryTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py new file mode 100755 index 0000000000..27dc0fb279 --- /dev/null +++ b/test/functional/wallet_create_tx.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class CreateTxWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + assert_equal(tx['locktime'], 0) + + self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block') + self.nodes[0].generate(1) + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + assert 0 < tx['locktime'] <= 201 + + +if __name__ == '__main__': + CreateTxWalletTest().main() diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py new file mode 100755 index 0000000000..9fd2650d78 --- /dev/null +++ b/test/functional/wallet_createwallet.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test createwallet arguments. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +class CreateWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + node.generate(1) # Leave IBD for sethdseed + + self.nodes[0].createwallet(wallet_name='w0') + w0 = node.get_wallet_rpc('w0') + address1 = w0.getnewaddress() + + self.log.info("Test disableprivatekeys creation.") + self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True) + w1 = node.get_wallet_rpc('w1') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) + w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info('Test that private keys cannot be imported') + addr = w0.getnewaddress('', 'legacy') + privkey = w0.dumpprivkey(addr) + assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) + result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) + assert(not result[0]['success']) + assert('warning' not in result[0]) + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Test blank creation with private keys disabled.") + self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True) + w2 = node.get_wallet_rpc('w2') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getrawchangeaddress) + w2.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info("Test blank creation with private keys enabled.") + self.nodes[0].createwallet(wallet_name='w3', disable_private_keys=False, blank=True) + w3 = node.get_wallet_rpc('w3') + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress) + # Import private key + w3.importprivkey(w0.dumpprivkey(address1)) + # Imported private keys are currently ignored by the keypool + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + # Set the seed + w3.sethdseed() + assert_equal(w3.getwalletinfo()['keypoolsize'], 1) + w3.getnewaddress() + w3.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys enabled and then encryption") + self.nodes[0].createwallet(wallet_name='w4', disable_private_keys=False, blank=True) + w4 = node.get_wallet_rpc('w4') + assert_equal(w4.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Encrypt the wallet. Nothing should change about the keypool + w4.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Now set a seed and it should work. Wallet should also be encrypted + w4.walletpassphrase('pass', 2) + w4.sethdseed() + w4.getnewaddress() + w4.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys disabled and then encryption") + self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True) + w5 = node.get_wallet_rpc('w5') + assert_equal(w5.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + # Encrypt the wallet + w5.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + +if __name__ == '__main__': + CreateWalletTest().main() diff --git a/test/functional/wallet_disableprivatekeys.py b/test/functional/wallet_disableprivatekeys.py deleted file mode 100755 index 34ff525255..0000000000 --- a/test/functional/wallet_disableprivatekeys.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test disable-privatekeys mode. -""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_raises_rpc_error, -) - - -class DisablePrivateKeysTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = False - self.num_nodes = 1 - self.supports_cli = True - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - node = self.nodes[0] - self.log.info("Test disableprivatekeys creation.") - self.nodes[0].createwallet('w1', True) - self.nodes[0].createwallet('w2') - w1 = node.get_wallet_rpc('w1') - w2 = node.get_wallet_rpc('w2') - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getnewaddress) - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) - w1.importpubkey(w2.getaddressinfo(w2.getnewaddress())['pubkey']) - -if __name__ == '__main__': - DisablePrivateKeysTest().main() diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index b1db1e4ab9..3f39654bb8 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -3,7 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" - import os from test_framework.test_framework import BitcoinTestFramework @@ -19,11 +18,12 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): Also check that the old hd_master is inactive """ with open(file_name, encoding='utf8') as inputfile: - found_addr = 0 + found_legacy_addr = 0 + found_p2sh_segwit_addr = 0 + found_bech32_addr = 0 found_script_addr = 0 found_addr_chg = 0 found_addr_rsv = 0 - witness_addr_ret = None hd_master_addr_ret = None for line in inputfile: # only read non comment lines @@ -60,14 +60,14 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): # count key types for addrObj in addrs: if addrObj['address'] == addr.split(",")[0] and addrObj['hdkeypath'] == keypath and keytype == "label=": - # a labeled entry in the wallet should contain both a native address - # and the p2sh-p2wpkh address that was added at wallet setup - if len(addr.split(",")) == 2: - addr_list = addr.split(",") - # the entry should be of the first key in the wallet - assert_equal(addrs[0]['address'], addr_list[0]) - witness_addr_ret = addr_list[1] - found_addr += 1 + if addr.startswith('m') or addr.startswith('n'): + # P2PKH address + found_legacy_addr += 1 + elif addr.startswith('2'): + # P2SH-segwit address + found_p2sh_segwit_addr += 1 + elif addr.startswith('bcrt1'): + found_bech32_addr += 1 break elif keytype == "change=1": found_addr_chg += 1 @@ -82,19 +82,19 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): found_script_addr += 1 break - return found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret, witness_addr_ret + return found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-keypool=90", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"]] + self.extra_args = [["-keypool=90", "-addresstype=legacy"]] self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() @@ -102,49 +102,54 @@ class WalletDumpTest(BitcoinTestFramework): wallet_unenc_dump = os.path.join(self.nodes[0].datadir, "wallet.unencrypted.dump") wallet_enc_dump = os.path.join(self.nodes[0].datadir, "wallet.encrypted.dump") - # generate 20 addresses to compare against the dump - # but since we add a p2sh-p2wpkh address for the first pubkey in the - # wallet, we will expect 21 addresses in the dump - test_addr_count = 20 + # generate 30 addresses to compare against the dump + # - 10 legacy P2PKH + # - 10 P2SH-segwit + # - 10 bech32 + test_addr_count = 10 addrs = [] - for i in range(0,test_addr_count): - addr = self.nodes[0].getnewaddress() - vaddr= self.nodes[0].getaddressinfo(addr) #required to get hd keypath - addrs.append(vaddr) - # Should be a no-op: - self.nodes[0].keypoolrefill() + for address_type in ['legacy', 'p2sh-segwit', 'bech32']: + for i in range(0, test_addr_count): + addr = self.nodes[0].getnewaddress(address_type=address_type) + vaddr = self.nodes[0].getaddressinfo(addr) # required to get hd keypath + addrs.append(vaddr) - # Test scripts dump by adding a P2SH witness and a 1-of-1 multisig address - witness_addr = self.nodes[0].addwitnessaddress(addrs[0]["address"], True) + # Test scripts dump by adding a 1-of-1 multisig address multisig_addr = self.nodes[0].addmultisigaddress(1, [addrs[1]["address"]])["address"] - script_addrs = [witness_addr, multisig_addr] + + # Refill the keypool. getnewaddress() refills the keypool *before* taking a key from + # the keypool, so the final call to getnewaddress leaves the keypool with one key below + # its capacity + self.nodes[0].keypoolrefill() # dump unencrypted wallet result = self.nodes[0].dumpwallet(wallet_unenc_dump) assert_equal(result['filename'], wallet_unenc_dump) - found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc, witness_addr_ret = \ - read_dump(wallet_unenc_dump, addrs, script_addrs, None) - assert_equal(found_addr, test_addr_count) # all keys must be in the dump - assert_equal(found_script_addr, 2) # all scripts must be in the dump + found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + read_dump(wallet_unenc_dump, addrs, [multisig_addr], None) + assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_script_addr, 1) # all scripts must be in the dump assert_equal(found_addr_chg, 0) # 0 blocks where mined assert_equal(found_addr_rsv, 90 * 2) # 90 keys plus 100% internal keys - assert_equal(witness_addr_ret, witness_addr) # p2sh-p2wsh address added to the first key - #encrypt wallet, restart, unlock and dump + # encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') self.nodes[0].walletpassphrase('test', 10) # Should be a no-op: self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) - found_addr, found_script_addr, found_addr_chg, found_addr_rsv, _, witness_addr_ret = \ - read_dump(wallet_enc_dump, addrs, script_addrs, hd_master_addr_unenc) - assert_equal(found_addr, test_addr_count) - assert_equal(found_script_addr, 2) + found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ + read_dump(wallet_enc_dump, addrs, [multisig_addr], hd_master_addr_unenc) + assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_script_addr, 1) assert_equal(found_addr_chg, 90 * 2) # old reserve keys are marked as change now assert_equal(found_addr_rsv, 90 * 2) - assert_equal(witness_addr_ret, witness_addr) # Overwriting should fail assert_raises_rpc_error(-8, "already exists", lambda: self.nodes[0].dumpwallet(wallet_enc_dump)) @@ -155,13 +160,13 @@ class WalletDumpTest(BitcoinTestFramework): # Make sure the address is not IsMine before import result = self.nodes[0].getaddressinfo(multisig_addr) - assert(result['ismine'] == False) + assert not result['ismine'] self.nodes[0].importwallet(wallet_unenc_dump) # Now check IsMine is true result = self.nodes[0].getaddressinfo(multisig_addr) - assert(result['ismine'] == True) + assert result['ismine'] if __name__ == '__main__': WalletDumpTest().main() diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index ab9ebed8d4..c514b7e0b4 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -31,12 +31,18 @@ class WalletEncryptionTest(BitcoinTestFramework): privkey = self.nodes[0].dumpprivkey(address) assert_equal(privkey[:1], "c") assert_equal(len(privkey), 52) + assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrase was called", self.nodes[0].walletpassphrase, 'ff', 1) + assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff') # Encrypt the wallet + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].encryptwallet, '') self.nodes[0].encryptwallet(passphrase) # Test that the wallet is encrypted assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff') + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1) + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff') # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 9d61483868..5452433acf 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -21,7 +21,7 @@ class WalletGroupTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] - self.rpc_timewait = 120 + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index f043a544fc..46462a16f3 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -46,11 +46,11 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): if self.call == Call.single: if self.data == Data.address: - response = self.try_rpc(self.node.importaddress, address=self.address["address"], rescan=rescan) + response = self.try_rpc(self.node.importaddress, address=self.address["address"], label=self.label, rescan=rescan) elif self.data == Data.pub: - response = self.try_rpc(self.node.importpubkey, pubkey=self.address["pubkey"], rescan=rescan) + response = self.try_rpc(self.node.importpubkey, pubkey=self.address["pubkey"], label=self.label, rescan=rescan) elif self.data == Data.priv: - response = self.try_rpc(self.node.importprivkey, privkey=self.key, rescan=rescan) + response = self.try_rpc(self.node.importprivkey, privkey=self.key, label=self.label, rescan=rescan) assert_equal(response, None) elif self.call in (Call.multiaddress, Call.multiscript): @@ -61,18 +61,32 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): "timestamp": timestamp + TIMESTAMP_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), "pubkeys": [self.address["pubkey"]] if self.data == Data.pub else [], "keys": [self.key] if self.data == Data.priv else [], + "label": self.label, "watchonly": self.data != Data.priv }], {"rescan": self.rescan in (Rescan.yes, Rescan.late_timestamp)}) assert_equal(response, [{"success": True}]) def check(self, txid=None, amount=None, confirmations=None): - """Verify that listreceivedbyaddress returns expected values.""" + """Verify that listtransactions/listreceivedbyaddress return expected values.""" + + txs = self.node.listtransactions(label=self.label, count=10000, include_watchonly=True) + assert_equal(len(txs), self.expected_txs) addresses = self.node.listreceivedbyaddress(minconf=0, include_watchonly=True, address_filter=self.address['address']) if self.expected_txs: assert_equal(len(addresses[0]["txids"]), self.expected_txs) if txid is not None: + tx, = [tx for tx in txs if tx["txid"] == txid] + assert_equal(tx["label"], self.label) + assert_equal(tx["address"], self.address["address"]) + assert_equal(tx["amount"], amount) + assert_equal(tx["category"], "receive") + assert_equal(tx["label"], self.label) + assert_equal(tx["txid"], txid) + assert_equal(tx["confirmations"], confirmations) + assert_equal("trusted" not in tx, True) + address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) assert_equal(address["amount"], self.expected_balance) @@ -122,21 +136,20 @@ class ImportRescanTest(BitcoinTestFramework): # Import keys with pruning disabled self.start_nodes(extra_args=[[]] * self.num_nodes) - super().import_deterministic_coinbase_privkeys() + for n in self.nodes: + n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') self.stop_nodes() self.start_nodes() for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) - def import_deterministic_coinbase_privkeys(self): - pass - def run_test(self): # Create one transaction on node 0 with a unique amount for # each possible type of wallet import RPC. for i, variant in enumerate(IMPORT_VARIANTS): - variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) + variant.label = "label {} {}".format(i, variant) + variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress(variant.label)) variant.key = self.nodes[1].dumpprivkey(variant.address["address"]) variant.initial_amount = 1 - (i + 1) / 64 variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) diff --git a/test/functional/wallet_import_with_label.py b/test/functional/wallet_import_with_label.py new file mode 100755 index 0000000000..a623b75606 --- /dev/null +++ b/test/functional/wallet_import_with_label.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the behavior of RPC importprivkey on set and unset labels of +addresses. + +It tests different cases in which an address is imported with importaddress +with or without a label and then its private key is imported with importprivkey +with and without a label. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet_util import test_address + + +class ImportWithLabel(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + """Main test logic""" + + self.log.info( + "Test importaddress with label and importprivkey without label." + ) + self.log.info("Import a watch-only address with a label.") + address = self.nodes[0].getnewaddress() + label = "Test Label" + self.nodes[1].importaddress(address, label) + test_address(self.nodes[1], + address, + iswatchonly=True, + ismine=False, + label=label) + + self.log.info( + "Import the watch-only address's private key without a " + "label and the address should keep its label." + ) + priv_key = self.nodes[0].dumpprivkey(address) + self.nodes[1].importprivkey(priv_key) + + test_address(self.nodes[1], + address, + label=label) + + self.log.info( + "Test importaddress without label and importprivkey with label." + ) + self.log.info("Import a watch-only address without a label.") + address2 = self.nodes[0].getnewaddress() + self.nodes[1].importaddress(address2) + test_address(self.nodes[1], + address2, + iswatchonly=True, + ismine=False, + label="") + + self.log.info( + "Import the watch-only address's private key with a " + "label and the address should have its label updated." + ) + priv_key2 = self.nodes[0].dumpprivkey(address2) + label2 = "Test Label 2" + self.nodes[1].importprivkey(priv_key2, label2) + + test_address(self.nodes[1], + address2, + label=label2) + + self.log.info("Test importaddress with label and importprivkey with label.") + self.log.info("Import a watch-only address with a label.") + address3 = self.nodes[0].getnewaddress() + label3_addr = "Test Label 3 for importaddress" + self.nodes[1].importaddress(address3, label3_addr) + test_address(self.nodes[1], + address3, + iswatchonly=True, + ismine=False, + label=label3_addr) + + self.log.info( + "Import the watch-only address's private key with a " + "label and the address should have its label updated." + ) + priv_key3 = self.nodes[0].dumpprivkey(address3) + label3_priv = "Test Label 3 for importprivkey" + self.nodes[1].importprivkey(priv_key3, label3_priv) + + test_address(self.nodes[1], + address3, + label=label3_priv) + + self.log.info( + "Test importprivkey won't label new dests with the same " + "label as others labeled dests for the same key." + ) + self.log.info("Import a watch-only legacy address with a label.") + address4 = self.nodes[0].getnewaddress() + label4_addr = "Test Label 4 for importaddress" + self.nodes[1].importaddress(address4, label4_addr) + test_address(self.nodes[1], + address4, + iswatchonly=True, + ismine=False, + label=label4_addr, + embedded=None) + + self.log.info( + "Import the watch-only address's private key without a " + "label and new destinations for the key should have an " + "empty label while the 'old' destination should keep " + "its label." + ) + priv_key4 = self.nodes[0].dumpprivkey(address4) + self.nodes[1].importprivkey(priv_key4) + embedded_addr = self.nodes[1].getaddressinfo(address4)['embedded']['address'] + + test_address(self.nodes[1], + embedded_addr, + label="") + test_address(self.nodes[1], + address4, + label=label4_addr) + + self.stop_nodes() + + +if __name__ == "__main__": + ImportWithLabel().main() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f78ff19791..7cce72b39f 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -2,9 +2,23 @@ # Copyright (c) 2014-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the importmulti RPC.""" - -from test_framework import script +"""Test the importmulti RPC. + +Test importmulti by generating keys on node0, importing the scriptPubKeys and +addresses on node1 and then testing the address info for the different address +variants. + +- `get_key()` and `get_multisig()` are called to generate keys on node0 and + return the privkeys, pubkeys and all variants of scriptPubKey and address. +- `test_importmulti()` is called to send an importmulti call to node1, test + success, and (if unsuccessful) test the error code and error message returned. +- `test_address()` is called to call getaddressinfo for an address on node1 + and test the values returned.""" + +from test_framework.script import ( + CScript, + OP_NOP, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -12,7 +26,11 @@ from test_framework.util import ( assert_raises_rpc_error, bytes_to_hex_str, ) - +from test_framework.wallet_util import ( + get_key, + get_multisig, + test_address, +) class ImportMultiTest(BitcoinTestFramework): def set_test_params(self): @@ -26,7 +44,19 @@ class ImportMultiTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - def run_test (self): + def test_importmulti(self, req, success, error_code=None, error_message=None, warnings=[]): + """Run importmulti and assert success""" + result = self.nodes[1].importmulti([req]) + observed_warnings = [] + if 'warnings' in result[0]: + observed_warnings = result[0]['warnings'] + assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) + assert_equal(result[0]['success'], success) + if error_code is not None: + assert_equal(result[0]['error']['code'], error_code) + assert_equal(result[0]['error']['message'], error_message) + + def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(1) self.nodes[1].generate(1) @@ -34,430 +64,567 @@ class ImportMultiTest(BitcoinTestFramework): node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - #Check only one address + # Check only one address assert_equal(node0_address1['ismine'], True) - #Node 1 sync test - assert_equal(self.nodes[1].getblockcount(),1) + # Node 1 sync test + assert_equal(self.nodes[1].getblockcount(), 1) - #Address Test - before import + # Address Test - before import address_info = self.nodes[1].getaddressinfo(node0_address1['address']) assert_equal(address_info['iswatchonly'], False) assert_equal(address_info['ismine'], False) - # RPC importmulti ----------------------------------------------- - # Bitcoin Address + # Bitcoin Address (implicit non-internal) self.log.info("Should import an address") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) - watchonly_address = address['address'] + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp, + ischange=False) + watchonly_address = key.p2pkh_addr watchonly_timestamp = timestamp self.log.info("Should not import an invalid address") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": "not valid address", - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Invalid address') + self.test_importmulti({"scriptPubKey": {"address": "not valid address"}, + "timestamp": "now"}, + success=False, + error_code=-5, + error_message='Invalid address \"not valid address\"') # ScriptPubKey + internal self.log.info("Should import a scriptPubKey with internal flag") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "internal": True - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "internal": True}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp, + ischange=True) + + # ScriptPubKey + internal + label + self.log.info("Should not allow a label to be specified when internal is true") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "internal": True, + "label": "Example label"}, + success=False, + error_code=-8, + error_message='Internal addresses should not have a label') # Nonstandard scriptPubKey + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal flag") - nonstandardScriptPubKey = address['scriptPubKey'] + bytes_to_hex_str(script.CScript([script.OP_NOP])) - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - - - # Address + Public key + !Internal + nonstandardScriptPubKey = key.p2pkh_script + bytes_to_hex_str(CScript([OP_NOP])) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) + + # Address + Public key + !Internal(explicit) self.log.info("Should import an address with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "pubkeys": [ address['pubkey'] ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) - + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "pubkeys": [key.pubkey], + "internal": False}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp) # ScriptPubKey + Public key + internal self.log.info("Should import a scriptPubKey with internal and with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "pubkeys": [ address['pubkey'] ], - "internal": True - }] - result = self.nodes[1].importmulti(request) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "pubkeys": [key.pubkey], + "internal": True}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp) # Nonstandard scriptPubKey + Public key + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal and with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - "pubkeys": [ address['pubkey'] ] - }] - result = self.nodes[1].importmulti(request) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now", + "pubkeys": [key.pubkey]}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) # Address + Private key + !watchonly self.log.info("Should import an address with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], True) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) self.log.info("Should not import an address with private key if is already imported") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -4) - assert_equal(result[0]['error']['message'], 'The wallet already contains the private key for this address or script') + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=False, + error_code=-4, + error_message='The wallet already contains the private key for this address or script ("' + key.p2pkh_script + '")') # Address + Private key + watchonly - self.log.info("Should not import an address with private key and with watchonly") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ], - "watchonly": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Incompatibility found between watchonly and keys') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) + self.log.info("Should import an address with private key and with watchonly") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey], + "watchonly": True}, + success=True, + warnings=["All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) # ScriptPubKey + Private key + internal self.log.info("Should import a scriptPubKey with internal and with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ], - "internal": True - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], True) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "keys": [key.privkey], + "internal": True}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) # Nonstandard scriptPubKey + Private key + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal and with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now", + "keys": [key.privkey]}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) # P2SH address - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['isscript'], True) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['timestamp'], timestamp) - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + isscript=True, + iswatchonly=True, + timestamp=timestamp) + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], False) - # P2SH + Redeem script - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['timestamp'], timestamp) - - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_addr, timestamp=timestamp, iswatchonly=True, ismine=False, solvable=True) + + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], True) - # P2SH + Redeem script + Private Keys + !Watchonly - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'], - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['timestamp'], timestamp) - - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script, + "keys": multisig.privkeys[0:2]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_addr, + timestamp=timestamp, + ismine=False, + iswatchonly=True, + solvable=True) + + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], True) # P2SH + Redeem script + Private Keys + Watchonly - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'], - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])], - "watchonly": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Incompatibility found between watchonly and keys') - + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script, + "keys": multisig.privkeys[0:2], + "watchonly": True}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + iswatchonly=True, + ismine=False, + solvable=True, + timestamp=timestamp) # Address + Public key + !Internal + Wrong pubkey - self.log.info("Should not import an address with a wrong public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "pubkeys": [ address2['pubkey'] ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Consistency check failed') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should not import an address with the wrong public key as non-solvable") + key = get_key(self.nodes[0]) + wrong_key = get_key(self.nodes[0]).pubkey + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "pubkeys": [wrong_key]}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # ScriptPubKey + Public key + internal + Wrong pubkey - self.log.info("Should not import a scriptPubKey with internal and with a wrong public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "pubkeys": [ address2['pubkey'] ], - "internal": True - }] - result = self.nodes[1].importmulti(request) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Consistency check failed') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import a scriptPubKey with internal and with a wrong public key as non-solvable") + key = get_key(self.nodes[0]) + wrong_key = get_key(self.nodes[0]).pubkey + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "pubkeys": [wrong_key], + "internal": True}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # Address + Private key + !watchonly + Wrong private key - self.log.info("Should not import an address with a wrong private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address2['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Consistency check failed') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import an address with a wrong private key as non-solvable") + key = get_key(self.nodes[0]) + wrong_privkey = get_key(self.nodes[0]).privkey + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [wrong_privkey]}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # ScriptPubKey + Private key + internal + Wrong private key - self.log.info("Should not import a scriptPubKey with internal and with a wrong private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address2['address']) ], - "internal": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Consistency check failed') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import a scriptPubKey with internal and with a wrong private key as non-solvable") + key = get_key(self.nodes[0]) + wrong_privkey = get_key(self.nodes[0]).privkey + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "keys": [wrong_privkey], + "internal": True}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # Importing existing watch only address with new timestamp should replace saved timestamp. assert_greater_than(timestamp, watchonly_timestamp) self.log.info("Should replace previously saved watch only timestamp.") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": watchonly_address, - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(watchonly_address) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) + self.test_importmulti({"scriptPubKey": {"address": watchonly_address}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + watchonly_address, + iswatchonly=True, + ismine=False, + timestamp=timestamp) watchonly_timestamp = timestamp - # restart nodes to check for proper serialization/deserialization of watch only address self.stop_nodes() self.start_nodes() - address_assert = self.nodes[1].getaddressinfo(watchonly_address) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], watchonly_timestamp) + test_address(self.nodes[1], + watchonly_address, + iswatchonly=True, + ismine=False, + timestamp=watchonly_timestamp) # Bad or missing timestamps self.log.info("Should throw on invalid or missing timestamp values") assert_raises_rpc_error(-3, 'Missing required timestamp field for key', - self.nodes[1].importmulti, [{ - "scriptPubKey": address['scriptPubKey'], - }]) + self.nodes[1].importmulti, [{"scriptPubKey": key.p2pkh_script}]) assert_raises_rpc_error(-3, 'Expected number or "now" timestamp value for key. got type string', - self.nodes[1].importmulti, [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "", - }]) + self.nodes[1].importmulti, [{ + "scriptPubKey": key.p2pkh_script, + "timestamp": "" + }]) + + # Import P2WPKH address as watch only + self.log.info("Should import a P2WPKH address as watch only") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2wpkh_addr, + iswatchonly=True, + solvable=False) + + # Import P2WPKH address with public key but no private key + self.log.info("Should import a P2WPKH address and public key as solvable but not spendable") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now", + "pubkeys": [key.pubkey]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2wpkh_addr, + ismine=False, + solvable=True) + + # Import P2WPKH address with key and check it is spendable + self.log.info("Should import a P2WPKH address with key") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2wpkh_addr, + iswatchonly=False, + ismine=True) + + # P2WSH multisig address without scripts or keys + multisig = get_multisig(self.nodes[0]) + self.log.info("Should import a p2wsh multisig as watch only without respective redeem script and private keys") + self.test_importmulti({"scriptPubKey": {"address": multisig.p2wsh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + solvable=False) + + # Same P2WSH multisig address as above, but now with witnessscript + private keys + self.log.info("Should import a p2wsh with respective witness script and private keys") + self.test_importmulti({"scriptPubKey": {"address": multisig.p2wsh_addr}, + "timestamp": "now", + "witnessscript": multisig.redeem_script, + "keys": multisig.privkeys}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + solvable=True, + ismine=True, + sigsrequired=2) + + # P2SH-P2WPKH address with no redeemscript or public or private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh without redeem script or keys") + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=False, + ismine=False) + + # P2SH-P2WPKH address + redeemscript + public key with no private key + self.log.info("Should import a p2sh-p2wpkh with respective redeem script and pubkey as solvable") + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now", + "redeemscript": key.p2sh_p2wpkh_redeem_script, + "pubkeys": [key.pubkey]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=False) + + # P2SH-P2WPKH address + redeemscript + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh with respective redeem script and private keys") + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now", + "redeemscript": key.p2sh_p2wpkh_redeem_script, + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True) + + # P2SH-P2WSH multisig + redeemscript with no private key + multisig = get_multisig(self.nodes[0]) + self.log.info("Should import a p2sh-p2wsh with respective redeem script but no private key") + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_p2wsh_addr}, + "timestamp": "now", + "redeemscript": multisig.p2wsh_script, + "witnessscript": multisig.redeem_script}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_p2wsh_addr, + solvable=True, + ismine=False) + + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True, + label="Descriptor import test") + + # Test ranged descriptor fails if range is not specified + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.log.info("Ranged descriptor import should fail without a specified range") + self.test_importmulti({"desc": desc, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor is ranged, please specify the range') + + # Test importing of a ranged descriptor without keys + self.log.info("Should import the ranged descriptor with specified range as solvable") + self.test_importmulti({"desc": desc, + "timestamp": "now", + "range": {"end": 1}}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + for address in addresses: + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True) + + # Test importing of a P2PKH address via descriptor + key = get_key(self.nodes[0]) + self.log.info("Should import a p2pkh address from descriptor") + self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + "timestamp": "now", + "label": "Descriptor import test"}, + True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + solvable=True, + ismine=False, + label="Descriptor import test") + + # Test import fails if both desc and scriptPubKey are provided + key = get_key(self.nodes[0]) + self.log.info("Import should fail if both scriptPubKey and desc are provided") + self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + "scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Both a descriptor and a scriptPubKey should not be provided.') + + # Test import fails if neither desc nor scriptPubKey are present + key = get_key(self.nodes[0]) + self.log.info("Import should fail if neither a descriptor nor a scriptPubKey are provided") + self.test_importmulti({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Either a descriptor or scriptPubKey must be provided.') + + # Test importing of a multisig via descriptor + key1 = get_key(self.nodes[0]) + key2 = get_key(self.nodes[0]) + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importmulti({"desc": "multi(1," + key1.pubkey + "," + key2.pubkey + ")", + "timestamp": "now"}, + success=True) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(self.nodes[1], + key1.p2pkh_addr, + ismine=False, + iswatchonly=False) if __name__ == '__main__': - ImportMultiTest ().main () + ImportMultiTest().main() diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index 26b181db33..78426018ef 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -81,7 +81,7 @@ class ImportPrunedFundsTest(BitcoinTestFramework): # Import with affiliated address with no rescan self.nodes[1].importaddress(address=address2, rescan=False) - self.nodes[1].importprunedfunds(rawtxn2, proof2) + self.nodes[1].importprunedfunds(rawtransaction=rawtxn2, txoutproof=proof2) assert [tx for tx in self.nodes[1].listtransactions(include_watchonly=True) if tx['txid'] == txnid2] # Import with private key with no rescan diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 51afa0cb1a..ceb9709712 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -73,11 +73,10 @@ class KeyPoolTest(BitcoinTestFramework): time.sleep(1.1) assert_equal(nodes[0].getwalletinfo()["unlocked_until"], 0) - # drain them by mining - nodes[0].generate(1) - nodes[0].generate(1) - nodes[0].generate(1) - assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].generate, 1) + # drain the keypool + for _ in range(3): + nodes[0].getnewaddress() + assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].getnewaddress) nodes[0].walletpassphrase('test', 100) nodes[0].keypoolrefill(100) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index f1a441c399..b7c8d3098d 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -24,8 +24,8 @@ from test_framework.util import ( class KeypoolRestoreTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 2 - self.extra_args = [[], ['-keypool=100']] + self.num_nodes = 4 + self.extra_args = [[], ['-keypool=100'], ['-keypool=100'], ['-keypool=100']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -40,32 +40,47 @@ class KeypoolRestoreTest(BitcoinTestFramework): shutil.copyfile(wallet_path, wallet_backup_path) self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 0, 2) + connect_nodes_bi(self.nodes, 0, 3) - self.log.info("Generate keys for wallet") - for _ in range(90): - addr_oldpool = self.nodes[1].getnewaddress() - for _ in range(20): - addr_extpool = self.nodes[1].getnewaddress() + for i, output_type in enumerate(["legacy", "p2sh-segwit", "bech32"]): - self.log.info("Send funds to wallet") - self.nodes[0].sendtoaddress(addr_oldpool, 10) - self.nodes[0].generate(1) - self.nodes[0].sendtoaddress(addr_extpool, 5) - self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.log.info("Generate keys for wallet with address type: {}".format(output_type)) + idx = i+1 + for _ in range(90): + addr_oldpool = self.nodes[idx].getnewaddress(address_type=output_type) + for _ in range(20): + addr_extpool = self.nodes[idx].getnewaddress(address_type=output_type) - self.log.info("Restart node with wallet backup") - self.stop_node(1) - shutil.copyfile(wallet_backup_path, wallet_path) - self.start_node(1, self.extra_args[1]) - connect_nodes_bi(self.nodes, 0, 1) - self.sync_all() + # Make sure we're creating the outputs we expect + address_details = self.nodes[idx].validateaddress(addr_extpool) + if i == 0: + assert(not address_details["isscript"] and not address_details["iswitness"]) + elif i == 1: + assert(address_details["isscript"] and not address_details["iswitness"]) + else: + assert(not address_details["isscript"] and address_details["iswitness"]) + + + self.log.info("Send funds to wallet") + self.nodes[0].sendtoaddress(addr_oldpool, 10) + self.nodes[0].generate(1) + self.nodes[0].sendtoaddress(addr_extpool, 5) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + self.log.info("Restart node with wallet backup") + self.stop_node(idx) + shutil.copyfile(wallet_backup_path, wallet_path) + self.start_node(idx, self.extra_args[idx]) + connect_nodes_bi(self.nodes, 0, idx) + self.sync_all() - self.log.info("Verify keypool is restored and balance is correct") - assert_equal(self.nodes[1].getbalance(), 15) - assert_equal(self.nodes[1].listtransactions()[0]['category'], "receive") - # Check that we have marked all keys up to the used keypool key as used - assert_equal(self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['hdkeypath'], "m/0'/0'/110'") + self.log.info("Verify keypool is restored and balance is correct") + assert_equal(self.nodes[idx].getbalance(), 15) + assert_equal(self.nodes[idx].listtransactions()[0]['category'], "receive") + # Check that we have marked all keys up to the used keypool key as used + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress())['hdkeypath'], "m/0'/0'/110'") if __name__ == '__main__': diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 8d7c77bb96..b71dae9f40 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -29,8 +29,8 @@ class WalletLabelsTest(BitcoinTestFramework): # Note each time we call generate, all generated coins go into # the same address, so we call twice to get two addresses w/50 each - node.generate(1) - node.generate(101) + node.generatetoaddress(nblocks=1, address=node.getnewaddress(label='coinbase')) + node.generatetoaddress(nblocks=101, address=node.getnewaddress(label='coinbase')) assert_equal(node.getbalance(), 100) # there should be 2 address groups @@ -42,8 +42,9 @@ class WalletLabelsTest(BitcoinTestFramework): linked_addresses = set() for address_group in address_groups: assert_equal(len(address_group), 1) - assert_equal(len(address_group[0]), 2) + assert_equal(len(address_group[0]), 3) assert_equal(address_group[0][1], 50) + assert_equal(address_group[0][2], 'coinbase') linked_addresses.add(address_group[0][0]) # send 50 from each address to a third address not in this wallet @@ -77,7 +78,7 @@ class WalletLabelsTest(BitcoinTestFramework): label.verify(node) # Check all labels are returned by listlabels. - assert_equal(node.listlabels(), [label.name for label in labels]) + assert_equal(node.listlabels(), sorted(['coinbase'] + [label.name for label in labels])) # Send a transaction to each label. for label in labels: diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 3485c4470f..011975e371 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -18,11 +18,6 @@ class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - def import_deterministic_coinbase_privkeys(self): - assert_equal(0, len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True))) - super().import_deterministic_coinbase_privkeys() - self.num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) - def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -31,6 +26,9 @@ class ReceivedByTest(BitcoinTestFramework): self.nodes[0].generate(1) sync_blocks(self.nodes) + # save the number of coinbase reward addresses so far + num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) + self.log.info("listreceivedbyaddress Test") # Send from node 0 to 1 @@ -68,11 +66,15 @@ class ReceivedByTest(BitcoinTestFramework): res = self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True, address_filter=addr) assert_array_result(res, {"address": addr}, expected) assert_equal(len(res), 1) + # Test for regression on CLI calls with address string (#14173) + cli_res = self.nodes[1].cli.listreceivedbyaddress(0, True, True, addr) + assert_array_result(cli_res, {"address": addr}, expected) + assert_equal(len(cli_res), 1) # Error on invalid address assert_raises_rpc_error(-4, "address_filter parameter was invalid", self.nodes[1].listreceivedbyaddress, minconf=0, include_empty=True, include_watchonly=True, address_filter="bamboozling") # Another address receive money res = self.nodes[1].listreceivedbyaddress(0, True, True) - assert_equal(len(res), 2 + self.num_cb_reward_addresses) # Right now 2 entries + assert_equal(len(res), 2 + num_cb_reward_addresses) # Right now 2 entries other_addr = self.nodes[1].getnewaddress() txid2 = self.nodes[0].sendtoaddress(other_addr, 0.1) self.nodes[0].generate(1) @@ -89,7 +91,7 @@ class ReceivedByTest(BitcoinTestFramework): assert_equal(len(res), 1) # Should be two entries though without filter res = self.nodes[1].listreceivedbyaddress(0, True, True) - assert_equal(len(res), 3 + self.num_cb_reward_addresses) # Became 3 entries + assert_equal(len(res), 3 + num_cb_reward_addresses) # Became 3 entries # Not on random addr other_addr = self.nodes[0].getnewaddress() # note on node[0]! just a random addr diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 5a17395abd..17f044bf65 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -25,12 +25,13 @@ def tx_from_hex(hexstring): class ListTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.enable_mocktime() def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.nodes[0].generate(1) # Get out of IBD + self.sync_all() # Simple send, 0 to 1: txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() @@ -97,9 +98,10 @@ class ListTransactionsTest(BitcoinTestFramework): txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1) self.nodes[1].generate(1) self.sync_all() - assert not [tx for tx in self.nodes[0].listtransactions(dummy="*", count=100, skip=0, include_watchonly=False) if "label" in tx and tx["label"] == "watchonly"] - txs = [tx for tx in self.nodes[0].listtransactions(dummy="*", count=100, skip=0, include_watchonly=True) if "label" in tx and tx['label'] == 'watchonly'] - assert_array_result(txs, {"category": "receive", "amount": Decimal("0.1")}, {"txid": txid}) + assert len(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=False)) == 0 + assert_array_result(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=True), + {"category": "receive", "amount": Decimal("0.1")}, + {"txid": txid, "label": "watchonly"}) self.run_rbf_opt_in_test() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 7d7c77638e..df778f57df 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -8,6 +8,7 @@ Verify that a bitcoind node can load multiple wallet files """ import os import shutil +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch @@ -46,9 +47,8 @@ class MultiWalletTest(BitcoinTestFramework): # create symlink to verify wallet directory path can be referenced # through symlink - if os.name != 'nt': - os.mkdir(wallet_dir('w7')) - os.symlink('w7', wallet_dir('w7_symlink')) + os.mkdir(wallet_dir('w7')) + os.symlink('w7', wallet_dir('w7_symlink')) # rename wallet.dat to make sure plain wallet file paths (as opposed to # directory paths) can be loaded @@ -69,11 +69,9 @@ class MultiWalletTest(BitcoinTestFramework): # w8 - to verify existing wallet file is loaded correctly # '' - to verify default wallet file is created correctly wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', ''] - if os.name == 'nt': - wallet_names.remove('w7_symlink') extra_args = ['-wallet={}'.format(n) for n in wallet_names] self.start_node(0, extra_args) - assert_equal(set(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), set(['', 'w3', 'w2', 'sub/w5', 'w7', 'w7', 'w1', 'w8', 'w'])) + assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8']) assert_equal(set(node.listwallets()), set(wallet_names)) @@ -99,9 +97,8 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) # should not initialize if wallet file is a symlink - if os.name != 'nt': - os.symlink('w8', wallet_dir('w8_symlink')) - self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX) + os.symlink('w8', wallet_dir('w8_symlink')) + self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX) # should not initialize if the specified walletdir does not exist self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') @@ -129,7 +126,7 @@ class MultiWalletTest(BitcoinTestFramework): self.start_node(0, ['-wallet=w4', '-wallet=w5']) assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") - w5.generate(1) + node.generatetoaddress(nblocks=1, address=w5.getnewaddress()) # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded os.rename(wallet_dir2, wallet_dir()) @@ -147,13 +144,13 @@ class MultiWalletTest(BitcoinTestFramework): self.restart_node(0, extra_args) - assert_equal(set(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), set(['', 'w3', 'w2', 'sub/w5', 'w7', 'w7', 'w8_copy', 'w1', 'w8', 'w'])) + assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy']) wallets = [wallet(w) for w in wallet_names] wallet_bad = wallet("bad") # check wallet names and balances - wallets[0].generate(1) + node.generatetoaddress(nblocks=1, address=wallets[0].getnewaddress()) for wallet_name, wallet in zip(wallet_names, wallets): info = wallet.getwalletinfo() assert_equal(info['immature_balance'], 50 if wallet is wallets[0] else 0) @@ -166,7 +163,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets - w1.generate(101) + node.generatetoaddress(nblocks=101, address=w1.getnewaddress()) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) @@ -175,7 +172,7 @@ class MultiWalletTest(BitcoinTestFramework): w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) w1.sendtoaddress(w4.getnewaddress(), 3) - w1.generate(1) + node.generatetoaddress(nblocks=1, address=w1.getnewaddress()) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) assert_equal(w4.getbalance(), 3) @@ -226,12 +223,18 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load duplicate wallets assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) + # Fail to load duplicate wallets by different ways (directory and filepath) + assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') + # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') + # Fail to load if one wallet is a copy of another, test this twice to make sure that we don't re-introduce #14304 + assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') + + # Fail to load if wallet file is a symlink - if os.name != 'nt': - assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') + assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) @@ -273,7 +276,11 @@ class MultiWalletTest(BitcoinTestFramework): assert 'w1' not in self.nodes[0].listwallets() # Successfully unload the wallet referenced by the request endpoint + # Also ensure unload works during walletpassphrase timeout + w2.encryptwallet('test') + w2.walletpassphrase('test', 1) w2.unloadwallet() + time.sleep(1.1) assert 'w2' not in self.nodes[0].listwallets() # Successfully unload all wallets @@ -287,7 +294,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_equal(self.nodes[0].listwallets(), ['w1']) assert_equal(w1.getwalletinfo()['walletname'], 'w1') - assert_equal(set(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), set(['', 'w3', 'w2', 'sub/w5', 'w7', 'w9', 'w7', 'w8_copy', 'w1', 'w8', 'w'])) + assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy', 'w9']) # Test backing up and restoring wallets self.log.info("Test wallet backup") @@ -308,6 +315,14 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], True) + # Test .walletlock file is closed + self.start_node(1) + wallet = os.path.join(self.options.tmpdir, 'my_wallet') + self.nodes[0].createwallet(wallet) + assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet) + self.nodes[0].unloadwallet(wallet) + self.nodes[1].loadwallet(wallet) + if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index d78c105c17..1c2e0a9cb7 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -65,7 +65,7 @@ class TxnMallTest(BitcoinTestFramework): # Construct a clone of tx1, to be malleated rawtx1 = self.nodes[0].getrawtransaction(txid1, 1) - clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"]}] + clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"], "sequence": rawtx1["vin"][0]["sequence"]}] clone_outputs = {rawtx1["vout"][0]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][0]["value"], rawtx1["vout"][1]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][1]["value"]} clone_locktime = rawtx1["locktime"] diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index b0d9f87958..c370ce0c04 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -26,8 +26,12 @@ SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def main(): - used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') - docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + if sys.version_info >= (3, 6): + used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') + docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + else: + used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() + docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index f1327469f3..4267f9fa0d 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -20,23 +20,23 @@ fi RET=0 PREV_BRANCH=`git name-rev --name-only HEAD` PREV_HEAD=`git rev-parse HEAD` -for i in `git rev-list --reverse $1`; do - if git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:"; then - git checkout --quiet $i^ || exit - SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" +for commit in `git rev-list --reverse $1`; do + if git rev-list -n 1 --pretty="%s" $commit | grep -q "^scripted-diff:"; then + git checkout --quiet $commit^ || exit + SCRIPT="`git rev-list --format=%b -n1 $commit | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" if test "x$SCRIPT" = "x"; then - echo "Error: missing script for: $i" + echo "Error: missing script for: $commit" echo "Failed" RET=1 else - echo "Running script for: $i" + echo "Running script for: $commit" echo "$SCRIPT" - eval "$SCRIPT" - git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1 + (eval "$SCRIPT") + git --no-pager diff --exit-code $commit && echo "OK" || (echo "Failed"; false) || RET=1 fi git reset --quiet --hard HEAD else - if git rev-list "--format=%b" -n1 $i | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then + if git rev-list "--format=%b" -n1 $commit | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then echo "Error: script block marker but no scripted-diff in title" echo "Failed" RET=1 diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 3972baed1d..be67cbed31 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -9,7 +9,7 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util -> chainparamsbase" + "chainparamsbase -> util/system -> chainparamsbase" "checkpoints -> validation -> checkpoints" "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" @@ -29,7 +29,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "validation -> validationinterface -> validation" "wallet/coincontrol -> wallet/wallet -> wallet/coincontrol" "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/rpcwallet -> wallet/wallet -> wallet/rpcwallet" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "policy/fees -> policy/policy -> validation -> policy/fees" "policy/rbf -> txmempool -> validation -> policy/rbf" diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 2fb35fd8ca..f5d4780b6d 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -16,8 +16,8 @@ FALSE_POSITIVES = [ ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), - ("src/util.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"), - ("src/util.cpp", "strprintf(COPYRIGHT_HOLDERS, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"), + ("src/util/system.cpp", "strprintf(COPYRIGHT_HOLDERS, COPYRIGHT_HOLDERS_SUBSTITUTION)"), ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), @@ -241,12 +241,11 @@ def count_format_specifiers(format_string): 4 """ assert(type(format_string) is str) + format_string = format_string.replace('%%', 'X') n = 0 in_specifier = False for i, char in enumerate(format_string): - if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%": - pass - elif char == "%": + if char == "%": in_specifier = True n += 1 elif char in "aAcdeEfFgGinopsuxX": diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index 2c443abf6b..c994ae3f4d 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -11,20 +11,20 @@ export LC_ALL=C FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - FatalError,0 - fprintf,1 - LogConnectFailure,1 - LogPrint,1 - LogPrintf,0 - printf,0 - snprintf,2 - sprintf,1 - strprintf,0 - vfprintf,1 - vprintf,1 - vsnprintf,1 - vsprintf,1 - WalletLogPrintf,0 + "FatalError,0" + "fprintf,1" + "LogConnectFailure,1" + "LogPrint,1" + "LogPrintf,0" + "printf,0" + "snprintf,2" + "sprintf,1" + "strprintf,0" + "vfprintf,1" + "vprintf,1" + "vsnprintf,1" + "vsprintf,1" + "WalletLogPrintf,0" ) EXIT_CODE=0 diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index f6d0fd382b..4b9e2615b6 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -50,7 +50,6 @@ EXPECTED_BOOST_INCLUDES=( boost/algorithm/string/classification.hpp boost/algorithm/string/replace.hpp boost/algorithm/string/split.hpp - boost/bind.hpp boost/chrono/chrono.hpp boost/date_time/posix_time/posix_time.hpp boost/filesystem.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index cbee437c91..1534d5ef68 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -2,36 +2,23 @@ export LC_ALL=C KNOWN_VIOLATIONS=( - "src/base58.cpp:.*isspace" "src/bitcoin-tx.cpp.*stoul" "src/bitcoin-tx.cpp.*trim_right" - "src/bitcoin-tx.cpp:.*atoi" - "src/core_read.cpp.*is_digit" "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" "src/qt/rpcconsole.cpp:.*atoi" - "src/qt/rpcconsole.cpp:.*isdigit" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/getarg_tests.cpp.*split" "src/torcontrol.cpp:.*atoi" "src/torcontrol.cpp:.*strtol" - "src/uint256.cpp:.*isspace" - "src/uint256.cpp:.*tolower" - "src/util.cpp:.*atoi" - "src/util.cpp:.*fprintf" - "src/util.cpp:.*tolower" - "src/utilmoneystr.cpp:.*isdigit" - "src/utilmoneystr.cpp:.*isspace" - "src/utilstrencodings.cpp:.*atoi" - "src/utilstrencodings.cpp:.*isspace" - "src/utilstrencodings.cpp:.*strtol" - "src/utilstrencodings.cpp:.*strtoll" - "src/utilstrencodings.cpp:.*strtoul" - "src/utilstrencodings.cpp:.*strtoull" - "src/utilstrencodings.h:.*atoi" + "src/util/strencodings.cpp:.*atoi" + "src/util/strencodings.cpp:.*strtol" + "src/util/strencodings.cpp:.*strtoul" + "src/util/strencodings.h:.*atoi" + "src/util/system.cpp:.*atoi" + "src/util/system.cpp:.*fprintf" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh new file mode 100755 index 0000000000..863caa9d5c --- /dev/null +++ b/test/lint/lint-python-dead-code.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Find dead Python code. + +export LC_ALL=C + +if ! command -v vulture > /dev/null; then + echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"" + exit 0 +fi + +vulture \ + --min-confidence 60 \ + --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,get_ecdh_key,get_privkey,is_compressed,is_fullyvalid,msg_generic,on_*,optionxform,restype,set_privkey,profile_with_perf" \ + $(git ls-files -- "*.py" ":(exclude)contrib/" ":(exclude)test/functional/data/invalid_txs.py") diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index d44a585294..f5b851aeab 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying @@ -8,76 +8,79 @@ export LC_ALL=C -# E101 indentation contains mixed spaces and tabs -# E112 expected an indented block -# E113 unexpected indentation -# E115 expected an indented block (comment) -# E116 unexpected indentation (comment) -# E125 continuation line with same indent as next logical line -# E129 visually indented line with same indent as next logical line -# E131 continuation line unaligned for hanging indent -# E133 closing bracket is missing indentation -# E223 tab before operator -# E224 tab after operator -# E242 tab after ',' -# E266 too many leading '#' for block comment -# E271 multiple spaces after keyword -# E272 multiple spaces before keyword -# E273 tab after keyword -# E274 tab before keyword -# E275 missing whitespace after keyword -# E304 blank lines found after function decorator -# E306 expected 1 blank line before a nested definition -# E401 multiple imports on one line -# E402 module level import not at top of file -# F403 'from foo_module import *' used; unable to detect undefined names -# F405 foo_function may be undefined, or defined from star imports: bar_module -# E502 the backslash is redundant between brackets -# E701 multiple statements on one line (colon) -# E702 multiple statements on one line (semicolon) -# E703 statement ends with a semicolon -# E714 test for object identity should be "is not" -# E721 do not compare types, use "isinstance()" -# E741 do not use variables named "l", "O", or "I" -# E742 do not define classes named "l", "O", or "I" -# E743 do not define functions named "l", "O", or "I" -# E901 SyntaxError: invalid syntax -# E902 TokenError: EOF in multi-line string -# F401 module imported but unused -# F402 import module from line N shadowed by loop variable -# F404 future import(s) name after other statements -# F406 "from module import *" only allowed at module level -# F407 an undefined __future__ feature name was imported -# F601 dictionary key name repeated with different values -# F602 dictionary key variable name repeated with different values -# F621 too many expressions in an assignment with star-unpacking -# F622 two or more starred expressions in an assignment (a, *b, *c = d) -# F631 assertion test is a tuple, which are always True -# F701 a break statement outside of a while or for loop -# F702 a continue statement outside of a while or for loop -# F703 a continue statement in a finally block in a loop -# F704 a yield or yield from statement outside of a function -# F705 a return statement with arguments inside a generator -# F706 a return statement outside of a function/method -# F707 an except: block as not the last exception handler -# F811 redefinition of unused name from line N -# F812 list comprehension redefines 'foo' from line N -# F821 undefined name 'Foo' -# F822 undefined name name in __all__ -# F823 local variable name … referenced before assignment -# F831 duplicate argument name in function definition -# F841 local variable 'foo' is assigned to but never used -# W191 indentation contains tabs -# W291 trailing whitespace -# W292 no newline at end of file -# W293 blank line contains whitespace -# W504 line break after binary operator -# W601 .has_key() is deprecated, use "in" -# W602 deprecated form of raising exception -# W603 "<>" is deprecated, use "!=" -# W604 backticks are deprecated, use "repr()" -# W605 invalid escape sequence "x" -# W606 'async' and 'await' are reserved keywords starting with Python 3.7 +enabled=( + E101 # indentation contains mixed spaces and tabs + E112 # expected an indented block + E113 # unexpected indentation + E115 # expected an indented block (comment) + E116 # unexpected indentation (comment) + E125 # continuation line with same indent as next logical line + E129 # visually indented line with same indent as next logical line + E131 # continuation line unaligned for hanging indent + E133 # closing bracket is missing indentation + E223 # tab before operator + E224 # tab after operator + E242 # tab after ',' + E266 # too many leading '#' for block comment + E271 # multiple spaces after keyword + E272 # multiple spaces before keyword + E273 # tab after keyword + E274 # tab before keyword + E275 # missing whitespace after keyword + E304 # blank lines found after function decorator + E306 # expected 1 blank line before a nested definition + E401 # multiple imports on one line + E402 # module level import not at top of file + E502 # the backslash is redundant between brackets + E701 # multiple statements on one line (colon) + E702 # multiple statements on one line (semicolon) + E703 # statement ends with a semicolon + E711 # comparison to None should be 'if cond is None:' + E714 # test for object identity should be "is not" + E721 # do not compare types, use "isinstance()" + E741 # do not use variables named "l", "O", or "I" + E742 # do not define classes named "l", "O", or "I" + E743 # do not define functions named "l", "O", or "I" + E901 # SyntaxError: invalid syntax + E902 # TokenError: EOF in multi-line string + F401 # module imported but unused + F402 # import module from line N shadowed by loop variable + F403 # 'from foo_module import *' used; unable to detect undefined names + F404 # future import(s) name after other statements + F405 # foo_function may be undefined, or defined from star imports: bar_module + F406 # "from module import *" only allowed at module level + F407 # an undefined __future__ feature name was imported + F601 # dictionary key name repeated with different values + F602 # dictionary key variable name repeated with different values + F621 # too many expressions in an assignment with star-unpacking + F622 # two or more starred expressions in an assignment (a, *b, *c = d) + F631 # assertion test is a tuple, which are always True + F701 # a break statement outside of a while or for loop + F702 # a continue statement outside of a while or for loop + F703 # a continue statement in a finally block in a loop + F704 # a yield or yield from statement outside of a function + F705 # a return statement with arguments inside a generator + F706 # a return statement outside of a function/method + F707 # an except: block as not the last exception handler + F811 # redefinition of unused name from line N + F812 # list comprehension redefines 'foo' from line N + F821 # undefined name 'Foo' + F822 # undefined name name in __all__ + F823 # local variable name … referenced before assignment + F831 # duplicate argument name in function definition + F841 # local variable 'foo' is assigned to but never used + W191 # indentation contains tabs + W291 # trailing whitespace + W292 # no newline at end of file + W293 # blank line contains whitespace + W504 # line break after binary operator + W601 # .has_key() is deprecated, use "in" + W602 # deprecated form of raising exception + W603 # "<>" is deprecated, use "!=" + W604 # backticks are deprecated, use "repr()" + W605 # invalid escape sequence "x" + W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 +) if ! command -v flake8 > /dev/null; then echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\"" @@ -87,4 +90,4 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then exit 0 fi -PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 "${@:-.}" +PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") "${@:-.}" diff --git a/test/lint/lint-rpc-help.sh b/test/lint/lint-rpc-help.sh new file mode 100755 index 0000000000..faac5d43e2 --- /dev/null +++ b/test/lint/lint-rpc-help.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check that all RPC help texts are generated by RPCHelpMan. + +export LC_ALL=C + +EXIT_CODE=0 + +# Assume that all multiline strings passed into a runtime_error are help texts. +# This is potentially fragile, but the linter is only temporary and can safely +# be removed early 2019. + +non_autogenerated_help=$(grep --perl-regexp --null-data --only-matching 'runtime_error\(\n\s*".*\\n"\n' $(git ls-files -- "*.cpp")) +if [[ ${non_autogenerated_help} != "" ]]; then + echo "Must use RPCHelpMan to generate the help for the following RPC methods:" + echo "${non_autogenerated_help}" + echo + EXIT_CODE=1 +fi +exit ${EXIT_CODE} diff --git a/test/lint/lint-python-shebang.sh b/test/lint/lint-shebang.sh index 4ff87f0bf7..fda22592d3 100755 --- a/test/lint/lint-python-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Shebang must use python3 (not python or python2) +# Assert expected shebang lines export LC_ALL=C EXIT_CODE=0 @@ -10,4 +10,11 @@ for PYTHON_FILE in $(git ls-files -- "*.py"); do EXIT_CODE=1 fi done +for SHELL_FILE in $(git ls-files -- "*.sh"); do + if [[ $(head -n 1 "${SHELL_FILE}") != "#!/usr/bin/env bash" && + $(head -n 1 "${SHELL_FILE}") != "#!/bin/sh" ]]; then + echo "Missing expected shebang \"#!/usr/bin/env bash\" or \"#!/bin/sh\" in ${SHELL_FILE}" + EXIT_CODE=1 + fi +done exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 9af3c10ed6..6f5e6546c5 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -13,7 +13,7 @@ export LC_ALL=C # respectively. So export LC_ALL=C is set as required by lint-shell-locale.sh # but unset here in case of running in Travis. if [ "$TRAVIS" = "true" ]; then - unset LC_ALL + unset LC_ALL fi if ! command -v shellcheck > /dev/null; then @@ -22,26 +22,26 @@ if ! command -v shellcheck > /dev/null; then fi # Disabled warnings: -# SC1087: Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet). -# SC1117: Backslash is literal in "\.". Prefer explicit escaping: "\\.". -# SC2001: See if you can use ${variable//search/replace} instead. -# SC2004: $/${} is unnecessary on arithmetic variables. -# SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'. -# SC2006: Use $(..) instead of legacy `..`. -# SC2016: Expressions don't expand in single quotes, use double quotes for that. -# SC2028: echo won't expand escape sequences. Consider printf. -# SC2046: Quote this to prevent word splitting. -# SC2048: Use "$@" (with quotes) to prevent whitespace problems. -# SC2066: Since you double quoted this, it will not word split, and the loop will only run once. -# SC2086: Double quote to prevent globbing and word splitting. -# SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'. -# SC2148: Tips depend on target shell and yours is unknown. Add a shebang. -# SC2162: read without -r will mangle backslashes. -# SC2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined. -# SC2166: Prefer [ p ] || [ q ] as [ p -o q ] is not well defined. -# SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. -# SC2206: Quote to prevent word splitting, or split robustly with mapfile or read -a. -# SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). -# SC2230: which is non-standard. Use builtin 'command -v' instead. -shellcheck -e SC1087,SC1117,SC2001,SC2004,SC2005,SC2006,SC2016,SC2028,SC2046,SC2048,SC2066,SC2086,SC2116,SC2148,SC2162,SC2166,SC2181,SC2206,SC2207,SC2230 \ +disabled=( + SC1087 # Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet). + SC2001 # See if you can use ${variable//search/replace} instead. + SC2004 # $/${} is unnecessary on arithmetic variables. + SC2005 # Useless echo? Instead of 'echo $(cmd)', just use 'cmd'. + SC2006 # Use $(..) instead of legacy `..`. + SC2016 # Expressions don't expand in single quotes, use double quotes for that. + SC2028 # echo won't expand escape sequences. Consider printf. + SC2046 # Quote this to prevent word splitting. + SC2048 # Use "$@" (with quotes) to prevent whitespace problems. + SC2066 # Since you double quoted this, it will not word split, and the loop will only run once. + SC2086 # Double quote to prevent globbing and word splitting. + SC2116 # Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'. + SC2162 # read without -r will mangle backslashes. + SC2166 # Prefer [ p ] {&&,||} [ q ] as [ p -{a,o} q ] is not well defined. + SC2181 # Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. + SC2206 # Quote to prevent word splitting, or split robustly with mapfile or read -a. + SC2207 # Prefer mapfile or read -a to split command output (or quote to avoid splitting). + SC2230 # which is non-standard. Use builtin 'command -v' instead. + SC2236 # Don't force -n instead of ! -z. +) +shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \ $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/') diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index beb7ec42f4..f318e19071 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -23,7 +23,7 @@ while getopts "?" opt; do done if [ -z "${TRAVIS_COMMIT_RANGE}" ]; then - if [ "$1" ]; then + if [ -n "$1" ]; then TRAVIS_COMMIT_RANGE="HEAD~$1...HEAD" else TRAVIS_COMMIT_RANGE="HEAD" diff --git a/test/sanitizer_suppressions/lsan b/test/sanitizer_suppressions/lsan new file mode 100644 index 0000000000..6f15c0f1d4 --- /dev/null +++ b/test/sanitizer_suppressions/lsan @@ -0,0 +1,6 @@ +# Suppress warnings triggered in dependencies +leak:libcrypto +leak:libqminimal +leak:libQt5Core +leak:libQt5Gui +leak:libQt5Widgets diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan new file mode 100644 index 0000000000..70eea34363 --- /dev/null +++ b/test/sanitizer_suppressions/tsan @@ -0,0 +1,16 @@ +# ThreadSanitizer suppressions +# ============================ + +# WalletBatch (unidentified deadlock) +deadlock:WalletBatch + +# Intentional deadlock in tests +deadlock:TestPotentialDeadLockDetected + +# Wildcard for all gui tests, should be replaced with non-wildcard suppressions +race:src/qt/test/* +deadlock:src/qt/test/* + +# External libraries +deadlock:libdb +race:libzmq diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan new file mode 100644 index 0000000000..d55119b266 --- /dev/null +++ b/test/sanitizer_suppressions/ubsan @@ -0,0 +1,34 @@ +alignment:move.h +alignment:prevector.h +bool:wallet/wallet.cpp +float-divide-by-zero:policy/fees.cpp +float-divide-by-zero:validation.cpp +float-divide-by-zero:wallet/wallet.cpp +unsigned-integer-overflow:arith_uint256.h +unsigned-integer-overflow:basic_string.h +unsigned-integer-overflow:bench/bench.h +unsigned-integer-overflow:bitcoin-tx.cpp +unsigned-integer-overflow:bloom.cpp +unsigned-integer-overflow:chain.cpp +unsigned-integer-overflow:chain.h +unsigned-integer-overflow:coded_stream.h +unsigned-integer-overflow:core_write.cpp +unsigned-integer-overflow:crypto/chacha20.cpp +unsigned-integer-overflow:crypto/ctaes/ctaes.c +unsigned-integer-overflow:crypto/ripemd160.cpp +unsigned-integer-overflow:crypto/sha1.cpp +unsigned-integer-overflow:crypto/sha256.cpp +unsigned-integer-overflow:crypto/sha512.cpp +unsigned-integer-overflow:hash.cpp +unsigned-integer-overflow:leveldb/db/log_reader.cc +unsigned-integer-overflow:leveldb/util/bloom.cc +unsigned-integer-overflow:leveldb/util/crc32c.h +unsigned-integer-overflow:leveldb/util/hash.cc +unsigned-integer-overflow:policy/fees.cpp +unsigned-integer-overflow:prevector.h +unsigned-integer-overflow:script/interpreter.cpp +unsigned-integer-overflow:stl_bvector.h +unsigned-integer-overflow:txmempool.cpp +unsigned-integer-overflow:util/strencodings.cpp +unsigned-integer-overflow:validation.cpp +vptr:fs.cpp diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 92fef30e13..7b1cc2b031 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -9,14 +9,9 @@ Runs automatically during `make check`. Can also be run manually.""" -from __future__ import division,print_function,unicode_literals - import argparse import binascii -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import difflib import json import logging diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py index 46e9fbc739..53058dc394 100755 --- a/test/util/rpcauth-test.py +++ b/test/util/rpcauth-test.py @@ -24,8 +24,8 @@ class TestRPCAuth(unittest.TestCase): self.rpcauth = importlib.import_module('rpcauth') def test_generate_salt(self): - self.assertLessEqual(len(self.rpcauth.generate_salt()), 32) - self.assertGreaterEqual(len(self.rpcauth.generate_salt()), 16) + for i in range(16, 32 + 1): + self.assertEqual(len(self.rpcauth.generate_salt(i)), i * 2) def test_generate_password(self): password = self.rpcauth.generate_password() @@ -34,7 +34,7 @@ class TestRPCAuth(unittest.TestCase): self.assertEqual(expected_password, password) def test_check_password_hmac(self): - salt = self.rpcauth.generate_salt() + salt = self.rpcauth.generate_salt(16) password = self.rpcauth.generate_password() password_hmac = self.rpcauth.password_to_hmac(salt, password) |