diff options
Diffstat (limited to 'test')
129 files changed, 4780 insertions, 2513 deletions
diff --git a/test/README.md b/test/README.md index 680f9bf9a6..9d4351b1de 100644 --- a/test/README.md +++ b/test/README.md @@ -18,7 +18,8 @@ request is opened. All sets of tests can also be run locally. # Running tests locally -Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. +Before tests can be run locally, Bitcoin Core must be built. See the [building instructions](/doc#building) for help. + ### Functional tests @@ -174,7 +175,28 @@ cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid gdb /home/example/bitcoind <pid> ``` -Note: gdb attach step may require `sudo` +Note: gdb attach step may require ptrace_scope to be modified, or `sudo` preceding the `gdb`. +See this link for considerations: https://www.kernel.org/doc/Documentation/security/Yama.txt + +##### Profiling + +An easy way to profile node performance during functional tests is provided +for Linux platforms using `perf`. + +Perf will sample the running node and will generate profile data in the node's +datadir. The profile data can then be presented using `perf report` or a graphical +tool like [hotspot](https://github.com/KDAB/hotspot). + +To generate a profile during test suite runs, use the `--perf` flag. + +To see render the output to text, run + +```sh +perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less +``` + +For ways to generate more granular profiles, see the README in +[test/functional](/test/functional). ### Util tests diff --git a/test/config.ini.in b/test/config.ini.in index 28abee2a3d..6b7ef70659 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/test/functional/README.md b/test/functional/README.md index d40052ac93..5e3009e6af 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -20,9 +20,13 @@ don't have test cases for. - Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/) - Use a python linter like flake8 before submitting PRs to catch common style nits (eg trailing whitespace, unused imports, etc) +- The oldest supported Python version is specified in [doc/dependencies.md](/doc/dependencies.md). + Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), + to prevent accidentally introducing modern syntax from an unsupported Python version. + The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that could lead to bugs and issues in the test code. -- Avoid wildcard imports where possible +- Avoid wildcard imports - Use a module-level docstring to describe what the test is testing, and how it is testing it. - When subclassing the BitcoinTestFramwork, place overrides for the @@ -39,6 +43,7 @@ don't have test cases for. - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` - use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` @@ -118,3 +123,36 @@ Helpers for script.py #### [test_framework/blocktools.py](test_framework/blocktools.py) Helper functions for creating blocks and transactions. + +### Benchmarking with perf + +An easy way to profile node performance during functional tests is provided +for Linux platforms using `perf`. + +Perf will sample the running node and will generate profile data in the node's +datadir. The profile data can then be presented using `perf report` or a graphical +tool like [hotspot](https://github.com/KDAB/hotspot). + +There are two ways of invoking perf: one is to use the `--perf` flag when +running tests, which will profile each node during the entire test run: perf +begins to profile when the node starts and ends when it shuts down. The other +way is the use the `profile_with_perf` context manager, e.g. + +```python +with node.profile_with_perf("send-big-msgs"): + # Perform activity on the node you're interested in profiling, e.g.: + for _ in range(10000): + node.p2p.send_message(some_large_message) +``` + +To see useful textual output, run + +```sh +perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | less +``` + +#### See also: + +- [Installing perf](https://askubuntu.com/q/50145) +- [Perf examples](http://www.brendangregg.com/perf.html) +- [Hotspot](https://github.com/KDAB/hotspot): a GUI for perf output analysis diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 3230d5cb6b..5bb3b5c094 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -2,7 +2,9 @@ """Combine logs from multiple bitcoin nodes as well as the test_framework log. This streams the combined log output to stdout. Use combine_logs.py > outputfile -to write to an outputfile.""" +to write to an outputfile. + +If no argument is provided, the most recent test directory will be used.""" import argparse from collections import defaultdict, namedtuple @@ -11,6 +13,13 @@ import itertools import os import re import sys +import tempfile + +# N.B.: don't import any local modules here - this script must remain executable +# without the parent module installed. + +# Should match same symbol in `test_framework.test_framework`. +TMPDIR_PREFIX = "bitcoin_func_test_" # Matches on the date format at the start of the log event TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?Z") @@ -19,22 +28,30 @@ LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) def main(): """Main function. Parses args, reads the log files and renders them as text or html.""" - - parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + 'testdir', nargs='?', default='', + help=('temporary test directory to combine logs from. ' + 'Defaults to the most recent')) parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)') parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2') - args, unknown_args = parser.parse_known_args() + args = parser.parse_args() if args.html and args.color: print("Only one out of --color or --html should be specified") sys.exit(1) - # There should only be one unknown argument - the path of the temporary test directory - if len(unknown_args) != 1: - print("Unexpected arguments" + str(unknown_args)) + testdir = args.testdir or find_latest_test_dir() + + if not testdir: + print("No test directories found") sys.exit(1) - log_events = read_logs(unknown_args[0]) + if not args.testdir: + print("Opening latest test directory: {}".format(testdir), file=sys.stderr) + + log_events = read_logs(testdir) print_logs(log_events, color=args.color, html=args.html) @@ -53,6 +70,29 @@ def read_logs(tmp_dir): return heapq.merge(*[get_log_events(source, f) for source, f in files]) + +def find_latest_test_dir(): + """Returns the latest tmpfile test directory prefix.""" + tmpdir = tempfile.gettempdir() + + def join_tmp(basename): + return os.path.join(tmpdir, basename) + + def is_valid_test_tmpdir(basename): + fullpath = join_tmp(basename) + return ( + os.path.isdir(fullpath) + and basename.startswith(TMPDIR_PREFIX) + and os.access(fullpath, os.R_OK) + ) + + testdir_paths = [ + join_tmp(name) for name in os.listdir(tmpdir) if is_valid_test_tmpdir(name) + ] + + return max(testdir_paths, key=os.path.getmtime) if testdir_paths else None + + def get_log_events(source, logfile): """Generator function that returns individual log events. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py new file mode 100644 index 0000000000..02deae92f3 --- /dev/null +++ b/test/functional/data/invalid_txs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Templates for constructing various sorts of invalid transactions. + +These templates (or an iterator over all of them) can be reused in different +contexts to test using a number of invalid transaction types. + +Hopefully this makes it easier to get coverage of a full variety of tx +validation checks through different interfaces (AcceptBlock, AcceptToMemPool, +etc.) without repeating ourselves. + +Invalid tx cases not covered here can be found by running: + + $ diff \ + <(grep -IREho "bad-txns[a-zA-Z-]+" src | sort -u) \ + <(grep -IEho "bad-txns[a-zA-Z-]+" test/functional/data/invalid_txs.py | sort -u) + +""" +import abc + +from test_framework.messages import CTransaction, CTxIn, CTxOut, COutPoint +from test_framework import script as sc +from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS + +basic_p2sh = sc.CScript([sc.OP_HASH160, sc.hash160(sc.CScript([sc.OP_0])), sc.OP_EQUAL]) + + +class BadTxTemplate: + """Allows simple construction of a certain kind of invalid tx. Base class to be subclassed.""" + __metaclass__ = abc.ABCMeta + + # The expected error code given by bitcoind upon submission of the tx. + reject_reason = "" + + # Only specified if it differs from mempool acceptance error. + block_reject_reason = "" + + # Do we expect to be disconnected after submitting this tx? + expect_disconnect = False + + # Is this tx considered valid when included in a block, but not for acceptance into + # the mempool (i.e. does it violate policy but not consensus)? + valid_in_block = False + + def __init__(self, *, spend_tx=None, spend_block=None): + self.spend_tx = spend_block.vtx[0] if spend_block else spend_tx + self.spend_avail = sum(o.nValue for o in self.spend_tx.vout) + self.valid_txin = CTxIn(COutPoint(self.spend_tx.sha256, 0), b"", 0xffffffff) + + @abc.abstractmethod + def get_tx(self, *args, **kwargs): + """Return a CTransaction that is invalid per the subclass.""" + pass + + +class OutputMissing(BadTxTemplate): + reject_reason = "bad-txns-vout-empty" + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.calc_sha256() + return tx + + +class InputMissing(BadTxTemplate): + reject_reason = "bad-txns-vin-empty" + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE] * 100))) + tx.calc_sha256() + return tx + + +class SizeTooSmall(BadTxTemplate): + reject_reason = "tx-size-small" + expect_disconnect = False + valid_in_block = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE]))) + tx.calc_sha256() + return tx + + +class BadInputOutpointIndex(BadTxTemplate): + # Won't be rejected - nonexistent outpoint index is treated as an orphan since the coins + # database can't distinguish between spent outpoints and outpoints which never existed. + reject_reason = None + expect_disconnect = False + + def get_tx(self): + num_indices = len(self.spend_tx.vin) + bad_idx = num_indices + 100 + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256, bad_idx), b"", 0xffffffff)) + tx.vout.append(CTxOut(0, basic_p2sh)) + tx.calc_sha256() + return tx + + +class DuplicateInput(BadTxTemplate): + reject_reason = 'bad-txns-inputs-duplicate' + expect_disconnect = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + +class NonexistentInput(BadTxTemplate): + reject_reason = None # Added as an orphan tx. + expect_disconnect = False + + def get_tx(self): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256 + 1, 0), b"", 0xffffffff)) + tx.vin.append(self.valid_txin) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + +class SpendTooMuch(BadTxTemplate): + reject_reason = 'bad-txns-in-belowout' + expect_disconnect = True + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_pub_key=basic_p2sh, amount=(self.spend_avail + 1)) + + +class SpendNegative(BadTxTemplate): + reject_reason = 'bad-txns-vout-negative' + expect_disconnect = True + + def get_tx(self): + return create_tx_with_script(self.spend_tx, 0, amount=-1) + + +class InvalidOPIFConstruction(BadTxTemplate): + reject_reason = "mandatory-script-verify-flag-failed (Invalid OP_IF construction)" + expect_disconnect = True + valid_in_block = True + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x64' * 35, + amount=(self.spend_avail // 2)) + + +class TooManySigops(BadTxTemplate): + reject_reason = "bad-txns-too-many-sigops" + block_reject_reason = "bad-blk-sigops, out-of-bounds SigOpCount" + expect_disconnect = False + + def get_tx(self): + lotsa_checksigs = sc.CScript([sc.OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) + return create_tx_with_script( + self.spend_tx, 0, + script_pub_key=lotsa_checksigs, + amount=1) + + +def iter_all_templates(): + """Iterate through all bad transaction template types.""" + return BadTxTemplate.__subclasses__() diff --git a/test/functional/example_test.py b/test/functional/example_test.py index be3544ee74..a2726763d0 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test @@ -13,7 +13,7 @@ is testing and *how* it's being tested # libraries then local imports). from collections import defaultdict -# Avoid wildcard * imports if possible +# Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv from test_framework.mininode import ( @@ -117,7 +117,7 @@ class ExampleTest(BitcoinTestFramework): # sync_all() should not include node2, since we're not expecting it to # sync. connect_nodes(self.nodes[0], 1) - self.sync_all([self.nodes[0:2]]) + self.sync_all(self.nodes[0:2]) # Use setup_nodes() to customize the node start behaviour (for example if # you don't want to start all nodes at the start of the test). @@ -141,7 +141,7 @@ class ExampleTest(BitcoinTestFramework): # Generating a block on one of the nodes will get us out of IBD blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] - self.sync_all([self.nodes[0:2]]) + self.sync_all(self.nodes[0:2]) # Notice above how we called an RPC by calling a method with the same # name on the node object. Notice also how we used a keyword argument diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 3d0467038d..b7814bf33e 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -32,7 +32,7 @@ Start three nodes: import time from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.key import CECKey +from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, COutPoint, @@ -104,9 +104,9 @@ class AssumeValidTest(BitcoinTestFramework): self.blocks = [] # Get a pubkey for the coinbase TXO - coinbase_key = CECKey() - coinbase_key.set_secretbytes(b"horsebattery") - coinbase_pubkey = coinbase_key.get_pubkey() + coinbase_key = ECKey() + coinbase_key.generate() + coinbase_pubkey = coinbase_key.get_pubkey().get_bytes() # Create the first block with a coinbase output to our key height = 1 @@ -180,7 +180,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(120) + p2p1.sync_with_ping(200) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8466f851ca..fdb60fb0e8 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -10,7 +10,13 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, ToHex from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, get_bip9_status, satoshi_round, sync_blocks +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + get_bip9_status, + satoshi_round, +) SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -63,7 +69,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC utxos = self.nodes[0].listunspent(0, 0) - assert(len(utxos) > 0) + assert len(utxos) > 0 utxo = utxos[0] @@ -253,7 +259,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].generate(1) cur_time += 600 - assert(tx2.hash in self.nodes[0].getrawmempool()) + assert tx2.hash in self.nodes[0].getrawmempool() test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=True) test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) @@ -264,23 +270,23 @@ class BIP68Test(BitcoinTestFramework): # Advance the time on the node so that we can test timelocks self.nodes[0].setmocktime(cur_time+600) self.nodes[0].generate(1) - assert(tx2.hash not in self.nodes[0].getrawmempool()) + assert tx2.hash not in self.nodes[0].getrawmempool() # Now that tx2 is not in the mempool, a sequence locked spend should # succeed tx3 = test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx3.hash in self.nodes[0].getrawmempool() self.nodes[0].generate(1) - assert(tx3.hash not in self.nodes[0].getrawmempool()) + assert tx3.hash not in self.nodes[0].getrawmempool() # One more test, this time using height locks tx4 = test_nonzero_locks(tx3, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx4.hash in self.nodes[0].getrawmempool()) + assert tx4.hash in self.nodes[0].getrawmempool() # Now try combining confirmed and unconfirmed inputs tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx5.hash not in self.nodes[0].getrawmempool()) + assert tx5.hash not in self.nodes[0].getrawmempool() utxos = self.nodes[0].listunspent() tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) @@ -299,8 +305,8 @@ class BIP68Test(BitcoinTestFramework): # If we invalidate the tip, tx3 should get added to the mempool, causing # tx4 to be removed (fails sequence-lock). self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - assert(tx4.hash not in self.nodes[0].getrawmempool()) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx4.hash not in self.nodes[0].getrawmempool() + assert tx3.hash in self.nodes[0].getrawmempool() # Now mine 2 empty blocks to reorg out the current tip (labeled tip-1 in # diagram above). @@ -319,8 +325,8 @@ class BIP68Test(BitcoinTestFramework): cur_time += 1 mempool = self.nodes[0].getrawmempool() - assert(tx3.hash not in mempool) - assert(tx2.hash in mempool) + assert tx3.hash not in mempool + assert tx2.hash in mempool # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) @@ -332,7 +338,7 @@ class BIP68Test(BitcoinTestFramework): # being run, then it's possible the test has activated the soft fork, and # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): - assert(get_bip9_status(self.nodes[0], 'csv')['status'] != 'active') + assert get_bip9_status(self.nodes[0], 'csv')['status'] != 'active' txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) tx1 = FromHex(CTransaction(), self.nodes[0].getrawtransaction(txid)) @@ -372,7 +378,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): @@ -385,7 +391,7 @@ class BIP68Test(BitcoinTestFramework): assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], "locked_in") self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], "active") - sync_blocks(self.nodes) + self.sync_blocks() # Use self.nodes[1] to test that version 2 transactions are standard. def test_version2_relay(self): diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index e50f67a345..61f705e239 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" @@ -7,8 +7,14 @@ import copy import struct import time -from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script, get_legacy_sigopcount_block -from test_framework.key import CECKey +from test_framework.blocktools import ( + create_block, + create_coinbase, + create_tx_with_script, + get_legacy_sigopcount_block, + MAX_BLOCK_SIGOPS, +) +from test_framework.key import ECKey from test_framework.messages import ( CBlock, COIN, @@ -45,8 +51,7 @@ from test_framework.script import ( ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal - -MAX_BLOCK_SIGOPS = 20000 +from data import invalid_txs # Use this class for tests that require behavior other than normal "mininode" behavior. # For now, it is used to serialize a bloated varint (b64). @@ -81,9 +86,9 @@ class FullBlockTest(BitcoinTestFramework): self.bootstrap_p2p() # Add one p2p connection to the node self.block_heights = {} - self.coinbase_key = CECKey() - self.coinbase_key.set_secretbytes(b"horsebattery") - self.coinbase_pubkey = self.coinbase_key.get_pubkey() + self.coinbase_key = ECKey() + self.coinbase_key.generate() + self.coinbase_pubkey = self.coinbase_key.get_pubkey().get_bytes() self.tip = None self.blocks = {} self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) @@ -93,18 +98,23 @@ class FullBlockTest(BitcoinTestFramework): # Create a new block b0 = self.next_block(0) self.save_spendable_output() - self.sync_blocks([b0]) + self.send_blocks([b0]) + + # These constants chosen specifically to trigger an immature coinbase spend + # at a certain time below. + NUM_BUFFER_BLOCKS_TO_GENERATE = 99 + NUM_OUTPUTS_TO_COLLECT = 33 # Allow the block to mature blocks = [] - for i in range(99): - blocks.append(self.next_block(5000 + i)) + for i in range(NUM_BUFFER_BLOCKS_TO_GENERATE): + blocks.append(self.next_block("maturitybuffer.{}".format(i))) self.save_spendable_output() - self.sync_blocks(blocks) + self.send_blocks(blocks) # collect spendable outputs now to avoid cluttering the code later on out = [] - for i in range(33): + for i in range(NUM_OUTPUTS_TO_COLLECT): out.append(self.get_spendable_output()) # Start by building a couple of blocks on top (which output is spent is @@ -116,7 +126,48 @@ class FullBlockTest(BitcoinTestFramework): b2 = self.next_block(2, spend=out[1]) self.save_spendable_output() - self.sync_blocks([b1, b2]) + self.send_blocks([b1, b2], timeout=4) + + # Select a txn with an output eligible for spending. This won't actually be spent, + # since we're testing submission of a series of blocks with invalid txns. + attempt_spend_tx = out[2] + + # Submit blocks for rejection, each of which contains a single transaction + # (aside from coinbase) which should be considered invalid. + for TxTemplate in invalid_txs.iter_all_templates(): + template = TxTemplate(spend_tx=attempt_spend_tx) + + if template.valid_in_block: + continue + + self.log.info("Reject block with invalid tx: %s", TxTemplate.__name__) + blockname = "for_invalid.%s" % TxTemplate.__name__ + badblock = self.next_block(blockname) + badtx = template.get_tx() + if TxTemplate != invalid_txs.InputMissing: + self.sign_tx(badtx, attempt_spend_tx) + else: + # Segwit is active in regtest at this point, so to deserialize a + # transaction without any inputs correctly, we set the outputs + # to an empty list. This is a hack, as the serialization of an + # empty list of outputs is deserialized as flags==0 and thus + # deserialization of the outputs is skipped. + # A policy check requires "loose" txs to be of a minimum size, + # so vtx is not set to be empty in the TxTemplate class and we + # only apply the workaround where txs are not "loose", i.e. in + # blocks. + # + # The workaround has the purpose that both sides calculate + # the same tx hash in the merkle tree + badtx.vout = [] + badtx.rehash() + badblock = self.update_block(blockname, [badtx]) + self.send_blocks( + [badblock], success=False, + reject_reason=(template.block_reject_reason or template.reject_reason), + reconnect=True, timeout=2) + + self.move_tip(2) # Fork like this: # @@ -128,7 +179,7 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(1) b3 = self.next_block(3, spend=out[1]) txout_b3 = b3.vtx[1] - self.sync_blocks([b3], False) + self.send_blocks([b3], False) # Now we add another block to make the alternative chain longer. # @@ -136,7 +187,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b3 (1) -> b4 (2) self.log.info("Reorg to a longer chain") b4 = self.next_block(4, spend=out[2]) - self.sync_blocks([b4]) + self.send_blocks([b4]) # ... and back to the first chain. # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -144,11 +195,11 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(2) b5 = self.next_block(5, spend=out[2]) self.save_spendable_output() - self.sync_blocks([b5], False) + self.send_blocks([b5], False) self.log.info("Reorg back to the original chain") b6 = self.next_block(6, spend=out[3]) - self.sync_blocks([b6], True) + self.send_blocks([b6], True) # Try to create a fork that double-spends # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -157,10 +208,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a chain with a double spend, even if it is longer") self.move_tip(5) b7 = self.next_block(7, spend=out[2]) - self.sync_blocks([b7], False) + self.send_blocks([b7], False) b8 = self.next_block(8, spend=out[4]) - self.sync_blocks([b8], False, reconnect=True) + self.send_blocks([b8], False, reconnect=True) # Try to create a block that has too much fee # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -169,7 +220,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block where the miner creates too much coinbase reward") self.move_tip(6) b9 = self.next_block(9, spend=out[4], additional_coinbase_value=1) - self.sync_blocks([b9], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b9], success=False, reject_reason='bad-cb-amount', reconnect=True) # Create a fork that ends in a block with too much fee (the one that causes the reorg) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -178,10 +229,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a chain where the miner creates too much coinbase reward, even if the chain is longer") self.move_tip(5) b10 = self.next_block(10, spend=out[3]) - self.sync_blocks([b10], False) + self.send_blocks([b10], False) b11 = self.next_block(11, spend=out[4], additional_coinbase_value=1) - self.sync_blocks([b11], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b11], success=False, reject_reason='bad-cb-amount', reconnect=True) # Try again, but with a valid fork first # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -194,7 +245,7 @@ class FullBlockTest(BitcoinTestFramework): b13 = self.next_block(13, spend=out[4]) self.save_spendable_output() b14 = self.next_block(14, spend=out[5], additional_coinbase_value=1) - self.sync_blocks([b12, b13, b14], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b12, b13, b14], success=False, reject_reason='bad-cb-amount', reconnect=True) # New tip should be b13. assert_equal(node.getbestblockhash(), b13.hash) @@ -208,12 +259,12 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(13) b15 = self.next_block(15, spend=out[5], script=lots_of_checksigs) self.save_spendable_output() - self.sync_blocks([b15], True) + self.send_blocks([b15], True) self.log.info("Reject a block with too many checksigs") too_many_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) b16 = self.next_block(16, spend=out[6], script=too_many_checksigs) - self.sync_blocks([b16], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b16], success=False, reject_reason='bad-blk-sigops', reconnect=True) # Attempt to spend a transaction created on a different fork # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -222,7 +273,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block with a spend from a re-org'ed out tx") self.move_tip(15) b17 = self.next_block(17, spend=txout_b3) - self.sync_blocks([b17], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b17], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a transaction created on a different fork (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -232,10 +283,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block with a spend from a re-org'ed out tx (on a forked chain)") self.move_tip(13) b18 = self.next_block(18, spend=txout_b3) - self.sync_blocks([b18], False) + self.send_blocks([b18], False) b19 = self.next_block(19, spend=out[6]) - self.sync_blocks([b19], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b19], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to spend a coinbase at depth too low # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -244,7 +295,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block spending an immature coinbase.") self.move_tip(15) b20 = self.next_block(20, spend=out[7]) - self.sync_blocks([b20], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') + self.send_blocks([b20], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') # Attempt to spend a coinbase at depth too low (on a fork this time) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -254,10 +305,10 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block spending an immature coinbase (on a forked chain)") self.move_tip(13) b21 = self.next_block(21, spend=out[6]) - self.sync_blocks([b21], False) + self.send_blocks([b21], False) b22 = self.next_block(22, spend=out[5]) - self.sync_blocks([b22], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') + self.send_blocks([b22], success=False, reject_reason='bad-txns-premature-spend-of-coinbase') # Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -275,7 +326,7 @@ class FullBlockTest(BitcoinTestFramework): b23 = self.update_block(23, [tx]) # Make sure the math above worked out to produce a max-sized block assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE) - self.sync_blocks([b23], True) + self.send_blocks([b23], True) self.save_spendable_output() self.log.info("Reject a block of size MAX_BLOCK_BASE_SIZE + 1") @@ -286,10 +337,10 @@ class FullBlockTest(BitcoinTestFramework): tx.vout = [CTxOut(0, script_output)] b24 = self.update_block(24, [tx]) assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE + 1) - self.sync_blocks([b24], success=False, reject_reason='bad-blk-length', reconnect=True) + self.send_blocks([b24], success=False, reject_reason='bad-blk-length', reconnect=True) b25 = self.next_block(25, spend=out[7]) - self.sync_blocks([b25], False) + self.send_blocks([b25], False) # Create blocks with a coinbase input script size out of range # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) @@ -304,11 +355,11 @@ class FullBlockTest(BitcoinTestFramework): # update_block causes the merkle root to get updated, even with no new # transactions, and updates the required state. b26 = self.update_block(26, []) - self.sync_blocks([b26], success=False, reject_reason='bad-cb-length', reconnect=True) + self.send_blocks([b26], success=False, reject_reason='bad-cb-length', reconnect=True) # Extend the b26 chain to make sure bitcoind isn't accepting b26 b27 = self.next_block(27, spend=out[7]) - self.sync_blocks([b27], False) + self.send_blocks([b27], False) # Now try a too-large-coinbase script self.move_tip(15) @@ -316,11 +367,11 @@ class FullBlockTest(BitcoinTestFramework): b28.vtx[0].vin[0].scriptSig = b'\x00' * 101 b28.vtx[0].rehash() b28 = self.update_block(28, []) - self.sync_blocks([b28], success=False, reject_reason='bad-cb-length', reconnect=True) + self.send_blocks([b28], success=False, reject_reason='bad-cb-length', reconnect=True) # Extend the b28 chain to make sure bitcoind isn't accepting b28 b29 = self.next_block(29, spend=out[7]) - self.sync_blocks([b29], False) + self.send_blocks([b29], False) # b30 has a max-sized coinbase scriptSig. self.move_tip(23) @@ -328,7 +379,7 @@ class FullBlockTest(BitcoinTestFramework): b30.vtx[0].vin[0].scriptSig = b'\x00' * 100 b30.vtx[0].rehash() b30 = self.update_block(30, []) - self.sync_blocks([b30], True) + self.send_blocks([b30], True) self.save_spendable_output() # b31 - b35 - check sigops of OP_CHECKMULTISIG / OP_CHECKMULTISIGVERIFY / OP_CHECKSIGVERIFY @@ -344,7 +395,7 @@ class FullBlockTest(BitcoinTestFramework): lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) b31 = self.next_block(31, spend=out[8], script=lots_of_multisigs) assert_equal(get_legacy_sigopcount_block(b31), MAX_BLOCK_SIGOPS) - self.sync_blocks([b31], True) + self.send_blocks([b31], True) self.save_spendable_output() # this goes over the limit because the coinbase has one sigop @@ -352,33 +403,33 @@ class FullBlockTest(BitcoinTestFramework): too_many_multisigs = CScript([OP_CHECKMULTISIG] * (MAX_BLOCK_SIGOPS // 20)) b32 = self.next_block(32, spend=out[9], script=too_many_multisigs) assert_equal(get_legacy_sigopcount_block(b32), MAX_BLOCK_SIGOPS + 1) - self.sync_blocks([b32], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b32], success=False, reject_reason='bad-blk-sigops', reconnect=True) # CHECKMULTISIGVERIFY self.log.info("Accept a block with the max number of OP_CHECKMULTISIGVERIFY sigops") self.move_tip(31) lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS - 1) // 20) + [OP_CHECKSIG] * 19) b33 = self.next_block(33, spend=out[9], script=lots_of_multisigs) - self.sync_blocks([b33], True) + self.send_blocks([b33], True) self.save_spendable_output() self.log.info("Reject a block with too many OP_CHECKMULTISIGVERIFY sigops") too_many_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * (MAX_BLOCK_SIGOPS // 20)) b34 = self.next_block(34, spend=out[10], script=too_many_multisigs) - self.sync_blocks([b34], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b34], success=False, reject_reason='bad-blk-sigops', reconnect=True) # CHECKSIGVERIFY self.log.info("Accept a block with the max number of OP_CHECKSIGVERIFY sigops") self.move_tip(33) lots_of_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS - 1)) b35 = self.next_block(35, spend=out[10], script=lots_of_checksigs) - self.sync_blocks([b35], True) + self.send_blocks([b35], True) self.save_spendable_output() self.log.info("Reject a block with too many OP_CHECKSIGVERIFY sigops") too_many_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS)) b36 = self.next_block(36, spend=out[11], script=too_many_checksigs) - self.sync_blocks([b36], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b36], success=False, reject_reason='bad-blk-sigops', reconnect=True) # Check spending of a transaction in a block which failed to connect # @@ -395,12 +446,12 @@ class FullBlockTest(BitcoinTestFramework): txout_b37 = b37.vtx[1] tx = self.create_and_sign_transaction(out[11], 0) b37 = self.update_block(37, [tx]) - self.sync_blocks([b37], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b37], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # attempt to spend b37's first non-coinbase tx, at which point b37 was still considered valid self.move_tip(35) b38 = self.next_block(38, spend=txout_b37) - self.sync_blocks([b38], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b38], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Check P2SH SigOp counting # @@ -450,7 +501,7 @@ class FullBlockTest(BitcoinTestFramework): b39_outputs += 1 b39 = self.update_block(39, []) - self.sync_blocks([b39], True) + self.send_blocks([b39], True) self.save_spendable_output() # Test sigops in P2SH redeem scripts @@ -477,7 +528,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b39.vtx[i].sha256, 0), b'')) # Note: must pass the redeem_script (not p2sh_script) to the signature hash function (sighash, err) = SignatureHash(redeem_script, tx, 1, SIGHASH_ALL) - sig = self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL])) + sig = self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL])) scriptSig = CScript([sig, redeem_script]) tx.vin[1].scriptSig = scriptSig @@ -492,7 +543,7 @@ class FullBlockTest(BitcoinTestFramework): tx.rehash() new_txs.append(tx) self.update_block(40, new_txs) - self.sync_blocks([b40], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b40], success=False, reject_reason='bad-blk-sigops', reconnect=True) # same as b40, but one less sigop self.log.info("Accept a block with the max number of P2SH sigops") @@ -505,7 +556,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b41_sigops_to_fill))) tx.rehash() self.update_block(41, [tx]) - self.sync_blocks([b41], True) + self.send_blocks([b41], True) # Fork off of b39 to create a constant base again # @@ -518,7 +569,7 @@ class FullBlockTest(BitcoinTestFramework): b43 = self.next_block(43, spend=out[13]) self.save_spendable_output() - self.sync_blocks([b42, b43], True) + self.send_blocks([b42, b43], True) # Test a number of really invalid scenarios # @@ -540,7 +591,7 @@ class FullBlockTest(BitcoinTestFramework): self.tip = b44 self.block_heights[b44.sha256] = height self.blocks[44] = b44 - self.sync_blocks([b44], True) + self.send_blocks([b44], True) self.log.info("Reject a block with a non-coinbase as the first tx") non_coinbase = self.create_tx(out[15], 0, 1) @@ -555,7 +606,7 @@ class FullBlockTest(BitcoinTestFramework): self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256] + 1 self.tip = b45 self.blocks[45] = b45 - self.sync_blocks([b45], success=False, reject_reason='bad-cb-missing', reconnect=True) + self.send_blocks([b45], success=False, reject_reason='bad-cb-missing', reconnect=True) self.log.info("Reject a block with no transactions") self.move_tip(44) @@ -570,7 +621,7 @@ class FullBlockTest(BitcoinTestFramework): self.tip = b46 assert 46 not in self.blocks self.blocks[46] = b46 - self.sync_blocks([b46], success=False, reject_reason='bad-blk-length', reconnect=True) + self.send_blocks([b46], success=False, reject_reason='bad-blk-length', reconnect=True) self.log.info("Reject a block with invalid work") self.move_tip(44) @@ -579,35 +630,35 @@ class FullBlockTest(BitcoinTestFramework): while b47.sha256 < target: b47.nNonce += 1 b47.rehash() - self.sync_blocks([b47], False, force_send=True, reject_reason='high-hash') + self.send_blocks([b47], False, force_send=True, reject_reason='high-hash') self.log.info("Reject a block with a timestamp >2 hours in the future") self.move_tip(44) b48 = self.next_block(48, solve=False) b48.nTime = int(time.time()) + 60 * 60 * 3 b48.solve() - self.sync_blocks([b48], False, force_send=True, reject_reason='time-too-new') + self.send_blocks([b48], False, force_send=True, reject_reason='time-too-new') self.log.info("Reject a block with invalid merkle hash") self.move_tip(44) b49 = self.next_block(49) b49.hashMerkleRoot += 1 b49.solve() - self.sync_blocks([b49], success=False, reject_reason='bad-txnmrklroot', reconnect=True) + self.send_blocks([b49], success=False, reject_reason='bad-txnmrklroot', reconnect=True) self.log.info("Reject a block with incorrect POW limit") self.move_tip(44) b50 = self.next_block(50) b50.nBits = b50.nBits - 1 b50.solve() - self.sync_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True) + self.send_blocks([b50], False, force_send=True, reject_reason='bad-diffbits', reconnect=True) self.log.info("Reject a block with two coinbase transactions") self.move_tip(44) b51 = self.next_block(51) cb2 = create_coinbase(51, self.coinbase_pubkey) b51 = self.update_block(51, [cb2]) - self.sync_blocks([b51], success=False, reject_reason='bad-cb-multiple', reconnect=True) + self.send_blocks([b51], success=False, reject_reason='bad-cb-multiple', reconnect=True) self.log.info("Reject a block with duplicate transactions") # Note: txns have to be in the right position in the merkle tree to trigger this error @@ -615,7 +666,7 @@ class FullBlockTest(BitcoinTestFramework): b52 = self.next_block(52, spend=out[15]) tx = self.create_tx(b52.vtx[1], 0, 1) b52 = self.update_block(52, [tx, tx]) - self.sync_blocks([b52], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b52], success=False, reject_reason='bad-txns-duplicate', reconnect=True) # Test block timestamps # -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) @@ -623,21 +674,21 @@ class FullBlockTest(BitcoinTestFramework): # self.move_tip(43) b53 = self.next_block(53, spend=out[14]) - self.sync_blocks([b53], False) + self.send_blocks([b53], False) self.save_spendable_output() self.log.info("Reject a block with timestamp before MedianTimePast") b54 = self.next_block(54, spend=out[15]) b54.nTime = b35.nTime - 1 b54.solve() - self.sync_blocks([b54], False, force_send=True, reject_reason='time-too-old') + self.send_blocks([b54], False, force_send=True, reject_reason='time-too-old') # valid timestamp self.move_tip(53) b55 = self.next_block(55, spend=out[15]) b55.nTime = b35.nTime self.update_block(55, []) - self.sync_blocks([b55], True) + self.send_blocks([b55], True) self.save_spendable_output() # Test Merkle tree malleability @@ -682,7 +733,7 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(len(b56.vtx), 3) b56 = self.update_block(56, [tx1]) assert_equal(b56.hash, b57.hash) - self.sync_blocks([b56], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b56], success=False, reject_reason='bad-txns-duplicate', reconnect=True) # b57p2 - a good block with 6 tx'es, don't submit until end self.move_tip(55) @@ -702,13 +753,13 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(b56p2.hash, b57p2.hash) assert_equal(len(b56p2.vtx), 6) b56p2 = self.update_block("b56p2", [tx3, tx4]) - self.sync_blocks([b56p2], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b56p2], success=False, reject_reason='bad-txns-duplicate', reconnect=True) self.move_tip("57p2") - self.sync_blocks([b57p2], True) + self.send_blocks([b57p2], True) self.move_tip(57) - self.sync_blocks([b57], False) # The tip is not updated because 57p2 seen first + self.send_blocks([b57], False) # The tip is not updated because 57p2 seen first self.save_spendable_output() # Test a few invalid tx types @@ -722,12 +773,12 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(57) b58 = self.next_block(58, spend=out[17]) tx = CTransaction() - assert(len(out[17].vout) < 42) + assert len(out[17].vout) < 42 tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), 0xffffffff)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() b58 = self.update_block(58, [tx]) - self.sync_blocks([b58], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b58], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # tx with output value > input value self.log.info("Reject a block with a transaction with outputs > inputs") @@ -735,12 +786,12 @@ class FullBlockTest(BitcoinTestFramework): b59 = self.next_block(59) tx = self.create_and_sign_transaction(out[17], 51 * COIN) b59 = self.update_block(59, [tx]) - self.sync_blocks([b59], success=False, reject_reason='bad-txns-in-belowout', reconnect=True) + self.send_blocks([b59], success=False, reject_reason='bad-txns-in-belowout', reconnect=True) # reset to good chain self.move_tip(57) b60 = self.next_block(60, spend=out[17]) - self.sync_blocks([b60], True) + self.send_blocks([b60], True) self.save_spendable_output() # Test BIP30 @@ -759,7 +810,7 @@ class FullBlockTest(BitcoinTestFramework): b61.vtx[0].rehash() b61 = self.update_block(61, []) assert_equal(b60.vtx[0].serialize(), b61.vtx[0].serialize()) - self.sync_blocks([b61], success=False, reject_reason='bad-txns-BIP30', reconnect=True) + self.send_blocks([b61], success=False, reject_reason='bad-txns-BIP30', reconnect=True) # Test tx.isFinal is properly rejected (not an exhaustive tx.isFinal test, that should be in data-driven transaction tests) # @@ -773,10 +824,10 @@ class FullBlockTest(BitcoinTestFramework): tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) - assert(tx.vin[0].nSequence < 0xffffffff) + assert tx.vin[0].nSequence < 0xffffffff tx.calc_sha256() b62 = self.update_block(62, [tx]) - self.sync_blocks([b62], success=False, reject_reason='bad-txns-nonfinal') + self.send_blocks([b62], success=False, reject_reason='bad-txns-nonfinal') # Test a non-final coinbase is also rejected # @@ -790,7 +841,7 @@ class FullBlockTest(BitcoinTestFramework): b63.vtx[0].vin[0].nSequence = 0xDEADBEEF b63.vtx[0].rehash() b63 = self.update_block(63, []) - self.sync_blocks([b63], success=False, reject_reason='bad-txns-nonfinal') + self.send_blocks([b63], success=False, reject_reason='bad-txns-nonfinal') # This checks that a block with a bloated VARINT between the block_header and the array of tx such that # the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint, @@ -824,7 +875,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) b64a = self.update_block("64a", [tx]) assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) - self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') + self.send_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently # resend the header message, it won't send us the getdata message again. Just @@ -840,7 +891,7 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE) self.blocks[64] = b64 b64 = self.update_block(64, []) - self.sync_blocks([b64], True) + self.send_blocks([b64], True) self.save_spendable_output() # Spend an output created in the block itself @@ -853,7 +904,7 @@ class FullBlockTest(BitcoinTestFramework): tx1 = self.create_and_sign_transaction(out[19], out[19].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 0) b65 = self.update_block(65, [tx1, tx2]) - self.sync_blocks([b65], True) + self.send_blocks([b65], True) self.save_spendable_output() # Attempt to spend an output created later in the same block @@ -866,7 +917,7 @@ class FullBlockTest(BitcoinTestFramework): tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 1) b66 = self.update_block(66, [tx2, tx1]) - self.sync_blocks([b66], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b66], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Attempt to double-spend a transaction created in a block # @@ -874,14 +925,14 @@ class FullBlockTest(BitcoinTestFramework): # \-> b67 (20) # # - self.log.info("Reject a block with a transaction double spending a transaction creted in the same block") + self.log.info("Reject a block with a transaction double spending a transaction created in the same block") self.move_tip(65) b67 = self.next_block(67) tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 1) tx3 = self.create_and_sign_transaction(tx1, 2) b67 = self.update_block(67, [tx1, tx2, tx3]) - self.sync_blocks([b67], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b67], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # More tests of block subsidy # @@ -900,14 +951,14 @@ class FullBlockTest(BitcoinTestFramework): b68 = self.next_block(68, additional_coinbase_value=10) tx = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue - 9) b68 = self.update_block(68, [tx]) - self.sync_blocks([b68], success=False, reject_reason='bad-cb-amount', reconnect=True) + self.send_blocks([b68], success=False, reject_reason='bad-cb-amount', reconnect=True) self.log.info("Accept a block claiming the correct subsidy in the coinbase transaction") self.move_tip(65) b69 = self.next_block(69, additional_coinbase_value=10) tx = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue - 10) self.update_block(69, [tx]) - self.sync_blocks([b69], True) + self.send_blocks([b69], True) self.save_spendable_output() # Test spending the outpoint of a non-existent transaction @@ -924,7 +975,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff)) tx.vout.append(CTxOut(1, b"")) b70 = self.update_block(70, [tx]) - self.sync_blocks([b70], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b70], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) # Test accepting an invalid block which has the same hash as a valid one (via merkle tree tricks) # @@ -949,10 +1000,10 @@ class FullBlockTest(BitcoinTestFramework): assert_equal(b72.sha256, b71.sha256) self.move_tip(71) - self.sync_blocks([b71], success=False, reject_reason='bad-txns-duplicate', reconnect=True) + self.send_blocks([b71], success=False, reject_reason='bad-txns-duplicate', reconnect=True) self.move_tip(72) - self.sync_blocks([b72], True) + self.send_blocks([b72], True) self.save_spendable_output() # Test some invalid scripts and MAX_BLOCK_SIGOPS @@ -987,7 +1038,7 @@ class FullBlockTest(BitcoinTestFramework): tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b73 = self.update_block(73, [tx]) assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS + 1) - self.sync_blocks([b73], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b73], success=False, reject_reason='bad-blk-sigops', reconnect=True) # b74/75 - if we push an invalid script element, all previous sigops are counted, # but sigops after the element are not counted. @@ -1011,7 +1062,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS + 4] = 0xff tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b74 = self.update_block(74, [tx]) - self.sync_blocks([b74], success=False, reject_reason='bad-blk-sigops', reconnect=True) + self.send_blocks([b74], success=False, reject_reason='bad-blk-sigops', reconnect=True) self.move_tip(72) b75 = self.next_block(75) @@ -1024,7 +1075,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS + 3] = 0xff tx = self.create_and_sign_transaction(out[22], 1, CScript(a)) b75 = self.update_block(75, [tx]) - self.sync_blocks([b75], True) + self.send_blocks([b75], True) self.save_spendable_output() # Check that if we push an element filled with CHECKSIGs, they are not counted @@ -1035,7 +1086,7 @@ class FullBlockTest(BitcoinTestFramework): a[MAX_BLOCK_SIGOPS - 1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs tx = self.create_and_sign_transaction(out[23], 1, CScript(a)) b76 = self.update_block(76, [tx]) - self.sync_blocks([b76], True) + self.send_blocks([b76], True) self.save_spendable_output() # Test transaction resurrection @@ -1060,40 +1111,40 @@ class FullBlockTest(BitcoinTestFramework): b77 = self.next_block(77) tx77 = self.create_and_sign_transaction(out[24], 10 * COIN) b77 = self.update_block(77, [tx77]) - self.sync_blocks([b77], True) + self.send_blocks([b77], True) self.save_spendable_output() b78 = self.next_block(78) tx78 = self.create_tx(tx77, 0, 9 * COIN) b78 = self.update_block(78, [tx78]) - self.sync_blocks([b78], True) + self.send_blocks([b78], True) b79 = self.next_block(79) tx79 = self.create_tx(tx78, 0, 8 * COIN) b79 = self.update_block(79, [tx79]) - self.sync_blocks([b79], True) + self.send_blocks([b79], True) # mempool should be empty assert_equal(len(self.nodes[0].getrawmempool()), 0) self.move_tip(77) b80 = self.next_block(80, spend=out[25]) - self.sync_blocks([b80], False, force_send=True) + self.send_blocks([b80], False, force_send=True) self.save_spendable_output() b81 = self.next_block(81, spend=out[26]) - self.sync_blocks([b81], False, force_send=True) # other chain is same length + self.send_blocks([b81], False, force_send=True) # other chain is same length self.save_spendable_output() b82 = self.next_block(82, spend=out[27]) - self.sync_blocks([b82], True) # now this chain is longer, triggers re-org + self.send_blocks([b82], True) # now this chain is longer, triggers re-org self.save_spendable_output() # now check that tx78 and tx79 have been put back into the peer's mempool mempool = self.nodes[0].getrawmempool() assert_equal(len(mempool), 2) - assert(tx78.hash in mempool) - assert(tx79.hash in mempool) + assert tx78.hash in mempool + assert tx79.hash in mempool # Test invalid opcodes in dead execution paths. # @@ -1110,7 +1161,7 @@ class FullBlockTest(BitcoinTestFramework): tx2.rehash() b83 = self.update_block(83, [tx1, tx2]) - self.sync_blocks([b83], True) + self.send_blocks([b83], True) self.save_spendable_output() # Reorg on/off blocks that have OP_RETURN in them (and try to spend them) @@ -1137,30 +1188,30 @@ class FullBlockTest(BitcoinTestFramework): tx5 = self.create_tx(tx1, 4, 0, CScript([OP_RETURN])) b84 = self.update_block(84, [tx1, tx2, tx3, tx4, tx5]) - self.sync_blocks([b84], True) + self.send_blocks([b84], True) self.save_spendable_output() self.move_tip(83) b85 = self.next_block(85, spend=out[29]) - self.sync_blocks([b85], False) # other chain is same length + self.send_blocks([b85], False) # other chain is same length b86 = self.next_block(86, spend=out[30]) - self.sync_blocks([b86], True) + self.send_blocks([b86], True) self.move_tip(84) b87 = self.next_block(87, spend=out[30]) - self.sync_blocks([b87], False) # other chain is same length + self.send_blocks([b87], False) # other chain is same length self.save_spendable_output() b88 = self.next_block(88, spend=out[31]) - self.sync_blocks([b88], True) + self.send_blocks([b88], True) self.save_spendable_output() # trying to spend the OP_RETURN output is rejected b89a = self.next_block("89a", spend=out[32]) tx = self.create_tx(tx1, 0, 0, CScript([OP_TRUE])) b89a = self.update_block("89a", [tx]) - self.sync_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) + self.send_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) self.log.info("Test a re-org of one week's worth of blocks (1088 blocks)") @@ -1169,7 +1220,7 @@ class FullBlockTest(BitcoinTestFramework): blocks = [] spend = out[32] for i in range(89, LARGE_REORG_SIZE + 89): - b = self.next_block(i, spend) + b = self.next_block(i, spend, version=4) tx = CTransaction() script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) @@ -1181,26 +1232,38 @@ class FullBlockTest(BitcoinTestFramework): self.save_spendable_output() spend = self.get_spendable_output() - self.sync_blocks(blocks, True, timeout=180) + self.send_blocks(blocks, True, timeout=480) chain1_tip = i # now create alt chain of same length self.move_tip(88) blocks2 = [] for i in range(89, LARGE_REORG_SIZE + 89): - blocks2.append(self.next_block("alt" + str(i))) - self.sync_blocks(blocks2, False, force_send=True) + blocks2.append(self.next_block("alt" + str(i), version=4)) + self.send_blocks(blocks2, False, force_send=True) # extend alt chain to trigger re-org - block = self.next_block("alt" + str(chain1_tip + 1)) - self.sync_blocks([block], True, timeout=180) + block = self.next_block("alt" + str(chain1_tip + 1), version=4) + self.send_blocks([block], True, timeout=480) # ... and re-org back to the first chain self.move_tip(chain1_tip) - block = self.next_block(chain1_tip + 1) - self.sync_blocks([block], False, force_send=True) - block = self.next_block(chain1_tip + 2) - self.sync_blocks([block], True, timeout=180) + block = self.next_block(chain1_tip + 1, version=4) + self.send_blocks([block], False, force_send=True) + block = self.next_block(chain1_tip + 2, version=4) + self.send_blocks([block], True, timeout=480) + + self.log.info("Reject a block with an invalid block header version") + b_v1 = self.next_block('b_v1', version=1) + self.send_blocks([b_v1], success=False, force_send=True, reject_reason='bad-version(0x00000001)') + + self.move_tip(chain1_tip + 2) + b_cb34 = self.next_block('b_cb34', version=4) + b_cb34.vtx[0].vin[0].scriptSig = b_cb34.vtx[0].vin[0].scriptSig[:-1] + b_cb34.vtx[0].rehash() + b_cb34.hashMerkleRoot = b_cb34.calc_merkle_root() + b_cb34.solve() + self.send_blocks([b_cb34], success=False, reject_reason='bad-cb-height', reconnect=True) # Helper methods ################ @@ -1221,7 +1284,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin[0].scriptSig = CScript() return (sighash, err) = SignatureHash(spend_tx.vout[0].scriptPubKey, tx, 0, SIGHASH_ALL) - tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))]) + tx.vin[0].scriptSig = CScript([self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL]))]) def create_and_sign_transaction(self, spend_tx, value, script=CScript([OP_TRUE])): tx = self.create_tx(spend_tx, 0, value, script) @@ -1229,7 +1292,7 @@ class FullBlockTest(BitcoinTestFramework): tx.rehash() return tx - def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True): + def next_block(self, number, spend=None, additional_coinbase_value=0, script=CScript([OP_TRUE]), solve=True, *, version=1): if self.tip is None: base_block_hash = self.genesis_hash block_time = int(time.time()) + 1 @@ -1242,11 +1305,11 @@ class FullBlockTest(BitcoinTestFramework): coinbase.vout[0].nValue += additional_coinbase_value coinbase.rehash() if spend is None: - block = create_block(base_block_hash, coinbase, block_time) + block = create_block(base_block_hash, coinbase, block_time, version=version) else: coinbase.vout[0].nValue += spend.vout[0].nValue - 1 # all but one satoshi to fees coinbase.rehash() - block = create_block(base_block_hash, coinbase, block_time) + block = create_block(base_block_hash, coinbase, block_time, version=version) tx = self.create_tx(spend, 0, 1, script) # spend 1 satoshi self.sign_tx(tx, spend) self.add_transactions_to_block(block, [tx]) @@ -1288,7 +1351,7 @@ class FullBlockTest(BitcoinTestFramework): self.blocks[block_number] = block return block - def bootstrap_p2p(self): + def bootstrap_p2p(self, timeout=10): """Add a P2P connection to the node. Helper to connect and wait for version handshake.""" @@ -1299,24 +1362,24 @@ class FullBlockTest(BitcoinTestFramework): # an INV for the next block and receive two getheaders - one for the # IBD and one for the INV. We'd respond to both and could get # unexpectedly disconnected if the DoS score for that error is 50. - self.nodes[0].p2p.wait_for_getheaders(timeout=5) + self.nodes[0].p2p.wait_for_getheaders(timeout=timeout) - def reconnect_p2p(self): + def reconnect_p2p(self, timeout=60): """Tear down and bootstrap the P2P connection to the node. The node gets disconnected several times in this test. This helper method reconnects the p2p and restarts the network thread.""" self.nodes[0].disconnect_p2ps() - self.bootstrap_p2p() + self.bootstrap_p2p(timeout=timeout) - def sync_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): + def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" self.nodes[0].p2p.send_blocks_and_test(blocks, self.nodes[0], success=success, reject_reason=reject_reason, force_send=force_send, timeout=timeout, expect_disconnect=reconnect) if reconnect: - self.reconnect_p2p() + self.reconnect_p2p(timeout=timeout) if __name__ == '__main__': diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index c170f510c8..3a4889bbe9 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -18,6 +18,8 @@ class BlocksdirTest(BitcoinTestFramework): def run_test(self): self.stop_node(0) + assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest", "blocks")) + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "blocks")) shutil.rmtree(self.nodes[0].datadir) initialize_datadir(self.options.tmpdir, 0) self.log.info("Starting with nonexistent blocksdir ...") diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 302a5ec1cb..b16eafccca 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -15,7 +15,6 @@ from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, O from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, hex_str_to_bytes, ) @@ -60,6 +59,7 @@ class BIP65Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -113,7 +113,7 @@ class BIP65Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index d87eabaa6d..2b93e3c24d 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -25,6 +25,10 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('-dash=1\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: -dash=1, options in configuration file must be specified without leading -') + with open(inc_conf_file_path, 'w', encoding='utf8') as conf: + conf.write("wallet=foo\n") + self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on regtest network when in [regtest] section.') + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 1: nono, if you intended to specify a negated option, use nono=1 instead') @@ -34,12 +38,28 @@ class ConfArgsTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('testnot.datadir=1\n[testnet]\n') + conf.write('server=1\nrpcuser=someuser\nmain.rpcpassword=some#pass') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 3, using # in rpcpassword can be ambiguous and should be avoided') + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided') + + inc_conf_file2_path = os.path.join(self.nodes[0].datadir, 'include2.conf') + with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: + conf.write('includeconf={}\n'.format(inc_conf_file2_path)) + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('testnot.datadir=1\n') + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('') # clear def run_test(self): self.stop_node(0) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index df79c4312c..887e9dafa3 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test activation of the first version bits soft fork. @@ -106,7 +106,7 @@ def send_generic_input_tx(node, coinbases, address): def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip68inputs) >= 16) + assert len(bip68inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip68inputs[i], address, amount=Decimal("49.98")) @@ -121,7 +121,7 @@ def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): def create_bip112txs(node, bip112inputs, varyOP_CSV, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip112inputs) >= 16) + assert len(bip112inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip112inputs[i], address, amount=Decimal("49.98")) @@ -168,7 +168,7 @@ class BIP68_112_113Test(BitcoinTestFramework): block.solve() return block - def sync_blocks(self, blocks, success=True): + def send_blocks(self, blocks, success=True): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" @@ -190,7 +190,7 @@ class BIP68_112_113Test(BitcoinTestFramework): self.log.info("Test that the csv softfork is DEFINED") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'defined') test_blocks = self.generate_blocks(61, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Advance from DEFINED to STARTED, height = 143") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'started') @@ -202,7 +202,7 @@ class BIP68_112_113Test(BitcoinTestFramework): test_blocks = self.generate_blocks(20, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, 536871169, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(24, 536936448, test_blocks) # 0x20010000 (signalling not) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Failed to advance past STARTED, height = 287") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'started') @@ -214,14 +214,14 @@ class BIP68_112_113Test(BitcoinTestFramework): test_blocks = self.generate_blocks(26, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, 536871169, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(10, 536936448, test_blocks) # 0x20010000 (signalling not) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Advanced from STARTED to LOCKED_IN, height = 431") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'locked_in') # Generate 140 more version 4 blocks test_blocks = self.generate_blocks(140, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # Inputs at height = 572 # @@ -264,7 +264,7 @@ class BIP68_112_113Test(BitcoinTestFramework): # 2 more version 4 blocks test_blocks = self.generate_blocks(2, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("Not yet advanced to ACTIVE, height = 574 (will activate for block 576, not 575)") assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'locked_in') @@ -318,7 +318,7 @@ class BIP68_112_113Test(BitcoinTestFramework): # try BIP 112 with seq=9 txs success_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v1)) success_txs.extend(all_rlt_txs(bip112txs_vary_OP_CSV_9_v1)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("Test version 2 txs") @@ -337,12 +337,12 @@ class BIP68_112_113Test(BitcoinTestFramework): # try BIP 112 with seq=9 txs success_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v2)) success_txs.extend(all_rlt_txs(bip112txs_vary_OP_CSV_9_v2)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # 1 more version 4 block to get us to height 575 so the fork should now be active for the next block test_blocks = self.generate_blocks(1, 4) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'active') self.log.info("Post-Soft Fork Tests.") @@ -354,74 +354,74 @@ class BIP68_112_113Test(BitcoinTestFramework): bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block bip113signed2 = sign_transaction(self.nodes[0], bip113tx_v2) for bip113tx in [bip113signed1, bip113signed2]: - self.sync_blocks([self.create_test_block([bip113tx])], success=False) + self.send_blocks([self.create_test_block([bip113tx])], success=False) # BIP 113 tests should now pass if the locktime is < MTP bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block bip113signed1 = sign_transaction(self.nodes[0], bip113tx_v1) bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block bip113signed2 = sign_transaction(self.nodes[0], bip113tx_v2) for bip113tx in [bip113signed1, bip113signed2]: - self.sync_blocks([self.create_test_block([bip113tx])]) + self.send_blocks([self.create_test_block([bip113tx])]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # Next block height = 580 after 4 blocks of random version test_blocks = self.generate_blocks(4, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) self.log.info("BIP 68 tests") self.log.info("Test version 1 txs - all should still pass") success_txs = [] success_txs.extend(all_rlt_txs(bip68txs_v1)) - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("Test version 2 txs") # All txs with SEQUENCE_LOCKTIME_DISABLE_FLAG set pass bip68success_txs = [tx['tx'] for tx in bip68txs_v2 if tx['sdf']] - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # All txs without flag fail as we are at delta height = 8 < 10 and delta time = 8 * 600 < 10 * 512 bip68timetxs = [tx['tx'] for tx in bip68txs_v2 if not tx['sdf'] and tx['stf']] for tx in bip68timetxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) bip68heighttxs = [tx['tx'] for tx in bip68txs_v2 if not tx['sdf'] and not tx['stf']] for tx in bip68heighttxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Advance one block to 581 test_blocks = self.generate_blocks(1, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # Height txs should fail and time txs should now pass 9 * 600 > 10 * 512 bip68success_txs.extend(bip68timetxs) - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) for tx in bip68heighttxs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Advance one block to 582 test_blocks = self.generate_blocks(1, 1234) - self.sync_blocks(test_blocks) + self.send_blocks(test_blocks) # All BIP 68 txs should pass bip68success_txs.extend(bip68heighttxs) - self.sync_blocks([self.create_test_block(bip68success_txs)]) + self.send_blocks([self.create_test_block(bip68success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) self.log.info("BIP 112 tests") self.log.info("Test version 1 txs") # -1 OP_CSV tx should fail - self.sync_blocks([self.create_test_block([bip112tx_special_v1])], success=False) + self.send_blocks([self.create_test_block([bip112tx_special_v1])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 1 txs should still pass success_txs = [tx['tx'] for tx in bip112txs_vary_OP_CSV_v1 if tx['sdf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if tx['sdf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is unset in argument to OP_CSV, version 1 txs should now fail @@ -430,18 +430,18 @@ class BIP68_112_113Test(BitcoinTestFramework): fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if not tx['sdf']] fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v1 if not tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) self.log.info("Test version 2 txs") # -1 OP_CSV tx should fail - self.sync_blocks([self.create_test_block([bip112tx_special_v2])], success=False) + self.send_blocks([self.create_test_block([bip112tx_special_v2])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 2 txs should pass (all sequence locks are met) success_txs = [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if tx['sdf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v2 if tx['sdf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # SEQUENCE_LOCKTIME_DISABLE_FLAG is unset in argument to OP_CSV for all remaining txs ## @@ -450,23 +450,23 @@ class BIP68_112_113Test(BitcoinTestFramework): fail_txs = all_rlt_txs(bip112txs_vary_nSequence_9_v2) fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_9_v2 if not tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in nSequence, tx should fail fail_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if tx['sdf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # If sequencelock types mismatch, tx should fail fail_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if not tx['sdf'] and tx['stf']] fail_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and tx['stf']] for tx in fail_txs: - self.sync_blocks([self.create_test_block([tx])], success=False) + self.send_blocks([self.create_test_block([tx])], success=False) # Remaining txs should pass, just test masking works properly success_txs = [tx['tx'] for tx in bip112txs_vary_nSequence_v2 if not tx['sdf'] and not tx['stf']] success_txs += [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and not tx['stf']] - self.sync_blocks([self.create_test_block(success_txs)]) + self.send_blocks([self.create_test_block(success_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # Additional test, of checking that comparison of two time types works properly @@ -476,7 +476,7 @@ class BIP68_112_113Test(BitcoinTestFramework): signtx = sign_transaction(self.nodes[0], tx) time_txs.append(signtx) - self.sync_blocks([self.create_test_block(time_txs)]) + self.send_blocks([self.create_test_block(time_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # TODO: Test empty stack fails diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 70d67aa53a..62062926a6 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. @@ -28,25 +28,19 @@ import errno import http.client import random -import sys import time from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, ToHex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, create_confirmed_utxos, hex_str_to_bytes -HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest] -try: - HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected) -except AttributeError: - pass class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = False # Need a bit of extra time for the nodes to start up for this test - self.rpc_timewait = 90 + self.rpc_timeout = 90 # Set -maxmempool=0 to turn off mempool memory sharing with dbcache # Set -rpcservertimeout=900 to reduce socket disconnects in this @@ -110,14 +104,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): try: self.nodes[node_index].submitblock(block) return True - except http.client.BadStatusLine as e: - # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error. - if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''": - self.log.debug("node %d submitblock raised exception: %s", node_index, e) - return False - else: - raise - except tuple(HTTP_DISCONNECT_ERRORS) as e: + except (http.client.CannotSendRequest, http.client.RemoteDisconnected) as e: self.log.debug("node %d submitblock raised exception: %s", node_index, e) return False except OSError as e: diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 9cbc1b39bd..7480e5c5ba 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -14,7 +14,6 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, wait_until, ) @@ -47,6 +46,7 @@ class BIP66Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1', '-enablebip61']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -102,7 +102,7 @@ class BIP66Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index b68e46adbc..a4b9f213a1 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" @@ -15,8 +15,6 @@ from test_framework.util import ( assert_greater_than_or_equal, connect_nodes, satoshi_round, - sync_blocks, - sync_mempools, ) # Construct 2 trivial P2SH's and the ScriptSigs that spend them @@ -65,7 +63,7 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee # the ScriptSig that will satisfy the ScriptPubKey. for inp in tx.vin: inp.scriptSig = SCRIPT_SIG[inp.prevout.n] - txid = from_node.sendrawtransaction(ToHex(tx), True) + txid = from_node.sendrawtransaction(hexstring=ToHex(tx), maxfeerate=0) unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) unconflist.append({"txid": txid, "vout": 1, "amount": amount}) @@ -95,7 +93,7 @@ def split_inputs(from_node, txins, txouts, initial_split=False): else: tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] completetx = ToHex(tx) - txid = from_node.sendrawtransaction(completetx, True) + txid = from_node.sendrawtransaction(hexstring=completetx, maxfeerate=0) txouts.append({"txid": txid, "vout": 0, "amount": half_change}) txouts.append({"txid": txid, "vout": 1, "amount": rem_change}) @@ -162,9 +160,9 @@ class EstimateFeeTest(BitcoinTestFramework): self.memutxo, Decimal("0.005"), min_fee, min_fee) tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee) / tx_kbytes) - sync_mempools(self.nodes[0:3], wait=.1) + self.sync_mempools(self.nodes[0:3], wait=.1) mined = mining_node.getblock(mining_node.generate(1)[0], True)["tx"] - sync_blocks(self.nodes[0:3], wait=.1) + self.sync_blocks(self.nodes[0:3], wait=.1) # update which txouts are confirmed newmem = [] for utx in self.memutxo: @@ -237,7 +235,7 @@ class EstimateFeeTest(BitcoinTestFramework): while len(self.nodes[1].getrawmempool()) > 0: self.nodes[1].generate(1) - sync_blocks(self.nodes[0:3], wait=.1) + self.sync_blocks(self.nodes[0:3], wait=.1) self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index d8083b2840..13cf951550 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" @@ -66,23 +66,7 @@ class NotificationsTest(BitcoinTestFramework): txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) assert_equal(sorted(txids_rpc), sorted(os.listdir(self.walletnotify_dir))) - # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. - self.log.info("test -alertnotify") - self.nodes[1].generatetoaddress(41, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - # Give bitcoind 10 seconds to write the alert notification - wait_until(lambda: len(os.listdir(self.alertnotify_dir)), timeout=10) - - for notify_file in os.listdir(self.alertnotify_dir): - os.remove(os.path.join(self.alertnotify_dir, notify_file)) - - # Mine more up-version blocks, should not get more alerts: - self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - self.log.info("-alertnotify should not continue notifying for more unknown version blocks") - assert_equal(len(os.listdir(self.alertnotify_dir)), 0) + # TODO: add test for `-alertnotify` large fork notifications if __name__ == '__main__': NotificationsTest().main() diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index eb76089d9c..a56c983ccc 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -18,7 +18,7 @@ from test_framework.blocktools import create_coinbase, create_block, create_tran from test_framework.messages import CTransaction from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str +from test_framework.util import assert_equal, assert_raises_rpc_error NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" @@ -27,7 +27,7 @@ def trueDummy(tx): newscript = [] for i in scriptSig: if (len(newscript) == 0): - assert(len(i) == 0) + assert len(i) == 0 newscript.append(b'\x51') else: newscript.append(i) @@ -64,17 +64,17 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] - txid1 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[0].serialize_with_witness()), True) + txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) - txid2 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[1].serialize_with_witness()), True) + txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) - txid3 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[2].serialize_with_witness()), True) + txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test1txs, False, True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) trueDummy(test2tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], False, True) @@ -83,19 +83,19 @@ class NULLDUMMYTest(BitcoinTestFramework): test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test5tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test5tx], True) self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") for i in test6txs: - self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) + self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test6txs, True, True) def block_submit(self, node, txs, witness=False, accept=False): @@ -108,7 +108,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 31d2ee8e13..be323d355e 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. @@ -44,6 +44,7 @@ RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports class ProxyTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + self.setup_clean_chain = True def setup_nodes(self): self.have_ipv6 = test_ipv6_local() @@ -94,7 +95,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv4 connection through node node.addnode("15.61.23.23:1234", "onetry") cmd = proxies[0].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"15.61.23.23") @@ -108,7 +109,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv6 connection through node node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"1233:3432:2434:2343:3234:2345:6546:4534") @@ -122,7 +123,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing onion connection through node node.addnode("bitcoinostk4e4re.onion:8333", "onetry") cmd = proxies[2].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"bitcoinostk4e4re.onion") assert_equal(cmd.port, 8333) @@ -134,7 +135,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing DNS name connection through node node.addnode("node.noumenon:8333", "onetry") cmd = proxies[3].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"node.noumenon") assert_equal(cmd.port, 8333) @@ -198,4 +199,3 @@ class ProxyTest(BitcoinTestFramework): if __name__ == '__main__': ProxyTest().main() - diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index c162f46d63..8fb7c49640 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -8,11 +8,20 @@ WARNING: This test uses 4GB of disk space. This test takes 30 mins or more (up to 2 hours) """ +import os +from test_framework.blocktools import create_coinbase +from test_framework.messages import CBlock, ToHex +from test_framework.script import CScript, OP_RETURN, OP_NOP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until - -import os +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, + wait_until, +) MIN_BLOCKS_TO_KEEP = 288 @@ -21,19 +30,59 @@ MIN_BLOCKS_TO_KEEP = 288 # compatible with pruning based on key creation time. TIMESTAMP_WINDOW = 2 * 60 * 60 +def mine_large_blocks(node, n): + # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN + # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase + # transaction but is consensus valid. + + # Get the block parameters for the first block + big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) + best_block = node.getblock(node.getbestblockhash()) + height = int(best_block["height"]) + 1 + try: + # Static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1 + except AttributeError: + mine_large_blocks.nTime = int(best_block["time"]) + 1 + previousblockhash = int(best_block["hash"], 16) + + for _ in range(n): + # Build the coinbase transaction (with large scriptPubKey) + coinbase_tx = create_coinbase(height) + coinbase_tx.vin[0].nSequence = 2 ** 32 - 1 + coinbase_tx.vout[0].scriptPubKey = big_script + coinbase_tx.rehash() + + # Build the block + block = CBlock() + block.nVersion = best_block["version"] + block.hashPrevBlock = previousblockhash + block.nTime = mine_large_blocks.nTime + block.nBits = int('207fffff', 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Submit to the node + node.submitblock(ToHex(block)) + + previousblockhash = block.sha256 + height += 1 + mine_large_blocks.nTime += 1 def calc_usage(blockdir): - return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) + return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timewait = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ @@ -55,10 +104,10 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[0], 4) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def setup_nodes(self): self.add_nodes(self.num_nodes, self.extra_args) @@ -69,23 +118,21 @@ class PruneTest(BitcoinTestFramework): def create_big_chain(self): # Start by creating some coinbases we can spend later self.nodes[1].generate(200) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MiB of data - for i in range(645): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 645) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def test_height_min(self): - if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): - raise AssertionError("blk00000.dat is missing, pruning too early") + assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) @@ -93,8 +140,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") + assert_greater_than(550, usage) def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks @@ -103,94 +149,72 @@ class PruneTest(BitcoinTestFramework): for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects - # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine - self.stop_node(0) - self.start_node(0, extra_args=self.full_node_default_args) + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[0], 2) # Mine 24 blocks in node 1 - for i in range(24): - if j == 0: - mine_large_block(self.nodes[1], self.utxo_cache_1) - else: - # Add node1's wallet transactions back to the mempool, to - # avoid the mined blocks from being too small. - self.nodes[1].resendwallettransactions() - self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + mine_large_blocks(self.nodes[1], 24) # Reorg back with 25 block chain from node 0 - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Create connections in the order so both nodes can see the reorg at the same time - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) - sync_blocks(self.nodes[0:3]) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + self.sync_blocks(self.nodes[0:3]) self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain - # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) - # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) - invalidheight = height-287 - badhash = self.nodes[1].getblockhash(invalidheight) - self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) - self.nodes[1].invalidateblock(badhash) + self.forkheight = height - 287 + self.forkhash = self.nodes[1].getblockhash(self.forkheight) + self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) - mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) while curhash != mainchainhash: self.nodes[1].invalidateblock(curhash) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) - assert(self.nodes[1].getblockcount() == invalidheight - 1) + assert self.nodes[1].getblockcount() == self.forkheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) - # Reboot node1 to clear those giant tx's from mempool - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) + # Disconnect node1 and generate the new chain + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[1], 2) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[2], 1) - sync_blocks(self.nodes[0:3], timeout=120) + connect_nodes(self.nodes[1], 2) + self.sync_blocks(self.nodes[0:3], timeout=120) self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) - - self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) - # Get node0's wallet transactions back in its mempool, to avoid the - # mined blocks from being too small. - self.nodes[0].resendwallettransactions() + self.log.info("Mine 220 more large blocks so we have requisite history") - for i in range(22): - # This can be slow, so do this in multiple RPC calls to avoid - # RPC timeouts. - self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects - sync_blocks(self.nodes[0:3], timeout=300) + mine_large_blocks(self.nodes[0], 220) usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") - - return invalidheight,badhash + assert_greater_than(550, usage) def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) + with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(pruning, no data)']): + self.nodes[2].verifychain(checklevel=4, nblocks=0) self.log.info("Will need to redownload block %d" % self.forkheight) # Verify that we have enough history to reorg back to the fork point @@ -217,17 +241,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert(self.nodes[0].getblockcount() == self.mainchainheight) - assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) + assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert(self.nodes[2].getbestblockhash() == goalbesthash) + assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned - assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode @@ -285,40 +309,32 @@ class PruneTest(BitcoinTestFramework): # height=100 too low to prune first block file so this is a no-op prune(100) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # Does nothing node.pruneblockchain(height(0)) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # height=500 should prune first file prune(500) - if has_block(0): - raise AssertionError("blk00000.dat is still there, should be pruned by now") - if not has_block(1): - raise AssertionError("blk00001.dat is missing when should still be there") + assert not has_block(0), "blk00000.dat is still there, should be pruned by now" + assert has_block(1), "blk00001.dat is missing when should still be there" # height=650 should prune second file prune(650) - if has_block(1): - raise AssertionError("blk00001.dat is still there, should be pruned by now") + assert not has_block(1), "blk00001.dat is still there, should be pruned by now" # height=1000 should not prune anything more, because tip-288 is in blk00002.dat. prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) - if not has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") + assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) node.generate(288) prune(1000) - if has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") - if has_block(3): - raise AssertionError("blk00003.dat is still there, should be pruned by now") + assert not has_block(2), "blk00002.dat is still there, should be pruned by now" + assert not has_block(3), "blk00003.dat is still there, should be pruned by now" - # stop node, start back up with auto-prune at 550MB, make sure still runs + # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) self.start_node(node_number, extra_args=["-prune=550"]) @@ -336,22 +352,15 @@ class PruneTest(BitcoinTestFramework): self.log.info("Syncing node 5 to test wallet") connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] - sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) #stop and start to trigger rescan + self.sync_blocks(nds, wait=5, timeout=300) + self.stop_node(5) # stop and start to trigger rescan self.start_node(5, extra_args=["-prune=550"]) self.log.info("Success") def run_test(self): - self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - self.log.info("Mining a big blockchain of 995 blocks") - - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] + self.log.info("Warning! This test requires 4GB of disk space") + self.log.info("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -392,11 +401,11 @@ class PruneTest(BitcoinTestFramework): # +...+(1044) &.. $...$(1319) # Save some current chain state for later use - self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.log.info("Check that we can survive a 288 block reorg still") - (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + self.reorg_test() # (1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index ff7c2b23bf..ccba547a1c 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -9,12 +9,12 @@ from decimal import Decimal from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, satoshi_round +from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round MAX_REPLACEMENT_LIMIT = 100 def txToHex(tx): - return bytes_to_hex_str(tx.serialize()) + return tx.serialize().hex() def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): """Create a txout with a given amount and scriptPubKey @@ -46,7 +46,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): signed_tx = node.signrawtransactionwithwallet(txToHex(tx2)) - txid = node.sendrawtransaction(signed_tx['hex'], True) + txid = node.sendrawtransaction(signed_tx['hex'], 0) # If requested, ensure txouts are confirmed. if confirmed: @@ -56,7 +56,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): new_size = len(node.getrawmempool()) # Error out if we have something stuck in the mempool, as this # would likely be a bug. - assert(new_size < mempool_size) + assert new_size < mempool_size mempool_size = new_size return COutPoint(int(txid, 16), 0) @@ -136,7 +136,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) self.sync_all() @@ -147,9 +147,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # This will raise an exception due to transaction replacement being disabled - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Extra 0.1 BTC fee tx1b = CTransaction() @@ -157,14 +157,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b.vout = [CTxOut(int(0.9 * COIN), CScript([b'b' * 35]))] tx1b_hex = txToHex(tx1b) # Replacement still disabled even with "enough fee" - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[1].sendrawtransaction, tx1b_hex, 0) # Works when enabled - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) mempool = self.nodes[0].getrawmempool() - assert (tx1a_txid not in mempool) - assert (tx1b_txid in mempool) + assert tx1a_txid not in mempool + assert tx1b_txid in mempool assert_equal(tx1b_hex, self.nodes[0].getrawtransaction(tx1b_txid)) @@ -188,7 +188,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vin = [CTxIn(prevout, nSequence=0)] tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] tx_hex = txToHex(tx) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) chain_txids.append(txid) prevout = COutPoint(int(txid, 16), 0) @@ -200,18 +200,18 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for doublespent_txid in chain_txids: - assert(doublespent_txid not in mempool) + assert doublespent_txid not in mempool def test_doublespend_tree(self): """Doublespend of a big tree of transactions""" @@ -236,8 +236,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vout = vout tx_hex = txToHex(tx) - assert(len(tx.serialize()) < 100000) - txid = self.nodes[0].sendrawtransaction(tx_hex, True) + assert len(tx.serialize()) < 100000 + txid = self.nodes[0].sendrawtransaction(tx_hex, 0) yield tx _total_txs[0] += 1 @@ -261,20 +261,20 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 1 BTC fee is enough dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] dbl_tx.vout = [CTxOut(initial_nValue - fee * n - 1 * COIN, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) - self.nodes[0].sendrawtransaction(dbl_tx_hex, True) + self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() for tx in tree_txs: tx.rehash() - assert (tx.hash not in mempool) + assert tx.hash not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -289,7 +289,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, CScript([1] * 35))] dbl_tx_hex = txToHex(dbl_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) for tx in tree_txs: tx.rehash() @@ -303,7 +303,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - self.nodes[0].sendrawtransaction(tx1a_hex, True) + self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. @@ -313,7 +313,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) def test_spends_of_conflicting_outputs(self): """Replacements that spend conflicting tx outputs are rejected""" @@ -324,7 +324,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(utxo1, nSequence=0)] tx1a.vout = [CTxOut(int(1.1 * COIN), CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) tx1a_txid = int(tx1a_txid, 16) @@ -336,14 +336,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. tx1b = CTransaction() tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] tx1b.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1b_hex = txToHex(tx1b) - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) tx1b_txid = int(tx1b_txid, 16) tx2 = CTransaction() @@ -353,7 +353,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" @@ -364,7 +364,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1.vin = [CTxIn(confirmed_utxo)] tx1.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1_hex = txToHex(tx1) - self.nodes[0].sendrawtransaction(tx1_hex, True) + self.nodes[0].sendrawtransaction(tx1_hex, 0) tx2 = CTransaction() tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] @@ -372,7 +372,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2_hex = txToHex(tx2) # This will raise an exception - assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, True) + assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) def test_too_many_replacements(self): """Replacements that evict too many transactions are rejected""" @@ -394,7 +394,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): splitting_tx.vout = outputs splitting_tx_hex = txToHex(splitting_tx) - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, True) + txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) txid = int(txid, 16) # Now spend each of those outputs individually @@ -403,7 +403,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] tx_i.vout = [CTxOut(split_value - fee, CScript([b'a' * 35]))] tx_i_hex = txToHex(tx_i) - self.nodes[0].sendrawtransaction(tx_i_hex, True) + self.nodes[0].sendrawtransaction(tx_i_hex, 0) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have @@ -418,14 +418,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx_hex = txToHex(double_tx) # This will raise an exception - assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, True) + assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass double_tx = CTransaction() double_tx.vin = inputs[0:-1] double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] double_tx_hex = txToHex(double_tx) - self.nodes[0].sendrawtransaction(double_tx_hex, True) + self.nodes[0].sendrawtransaction(double_tx_hex, 0) def test_opt_in(self): """Replacing should only work if orig tx opted in""" @@ -436,7 +436,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0xffffffff)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # This transaction isn't shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) @@ -448,7 +448,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -457,7 +457,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, True) + tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Still shouldn't be able to double-spend tx2b = CTransaction() @@ -466,7 +466,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # This will raise an exception - assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now create a new transaction that spends from tx1a and tx2a # opt-in on one of the inputs @@ -481,7 +481,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3a.vout = [CTxOut(int(0.9*COIN), CScript([b'c'])), CTxOut(int(0.9*COIN), CScript([b'd']))] tx3a_hex = txToHex(tx3a) - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, True) + tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) @@ -496,10 +496,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx3c.vout = [CTxOut(int(0.5 * COIN), CScript([b'f' * 35]))] tx3c_hex = txToHex(tx3c) - self.nodes[0].sendrawtransaction(tx3b_hex, True) + self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, True) + self.nodes[0].sendrawtransaction(tx3c_hex, 0) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -512,7 +512,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx1a_hex = txToHex(tx1a) - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) + tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) # Higher fee, but the actual fee per KB is much lower. tx1b = CTransaction() @@ -521,15 +521,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1b_hex = txToHex(tx1b) # Verify tx1b cannot replace tx1a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # Use prioritisetransaction to set tx1a's fee to 0. self.nodes[0].prioritisetransaction(txid=tx1a_txid, fee_delta=int(-0.1*COIN)) # Now tx1b should be able to replace tx1a - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) + tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - assert(tx1b_txid in self.nodes[0].getrawmempool()) + assert tx1b_txid in self.nodes[0].getrawmempool() # 2. Check that absolute fee checks use modified fee. tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -538,7 +538,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] tx2a.vout = [CTxOut(1 * COIN, CScript([b'a' * 35]))] tx2a_hex = txToHex(tx2a) - self.nodes[0].sendrawtransaction(tx2a_hex, True) + self.nodes[0].sendrawtransaction(tx2a_hex, 0) # Lower fee, but we'll prioritise it tx2b = CTransaction() @@ -548,15 +548,15 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2b_hex = txToHex(tx2b) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, True) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) # Now prioritise tx2b to have a higher modified fee self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1*COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) - assert(tx2b_txid in self.nodes[0].getrawmempool()) + assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): us0 = self.nodes[0].listunspent()[0] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 7098a03f1e..2d4dd96a1d 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -18,7 +18,13 @@ from test_framework.blocktools import witness_script, send_to_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, sha256, ToHex from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, + hex_str_to_bytes, + try_rpc, +) NODE_0 = 0 NODE_2 = 2 @@ -38,6 +44,8 @@ def find_spendable_utxo(node, min_value): raise AssertionError("Unspent output equal or higher than %s not found" % min_value) +txs_mined = {} # txindex from txid to blockhash + class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -74,13 +82,13 @@ class SegWitTest(BitcoinTestFramework): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) block = node.generate(1) assert_equal(len(node.getblock(block[0])["tx"]), 2) - sync_blocks(self.nodes) + self.sync_blocks() def skip_mine(self, node, txid, sign, redeem_script=""): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) block = node.generate(1) assert_equal(len(node.getblock(block[0])["tx"]), 1) - sync_blocks(self.nodes) + self.sync_blocks() def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) @@ -90,18 +98,18 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tmpl = self.nodes[0].getblocktemplate({}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 + tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -129,7 +137,7 @@ class SegWitTest(BitcoinTestFramework): p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) self.nodes[0].generate(1) # block 163 - sync_blocks(self.nodes) + self.sync_blocks() # Make sure all nodes recognize the transactions as theirs assert_equal(self.nodes[0].getbalance(), balance_presetup - 60 * 50 + 20 * Decimal("49.999") + 50) @@ -137,7 +145,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) self.nodes[0].generate(260) # block 423 - sync_blocks(self.nodes) + self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) # block 424 @@ -153,10 +161,10 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - block = self.nodes[2].generate(1) # block 432 (first block with new rules; 432 = 144 * 3) - sync_blocks(self.nodes) + blockhash = self.nodes[2].generate(1)[0] # block 432 (first block with new rules; 432 = 144 * 3) + self.sync_blocks() assert_equal(len(self.nodes[2].getrawmempool()), 0) - segwit_tx_list = self.nodes[2].getblock(block[0])["tx"] + segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) self.log.info("Verify default node can't accept txs with missing witness") @@ -170,15 +178,16 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) - assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) - for i in range(len(segwit_tx_list)): - tx = FromHex(CTransaction(), self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[2].getrawtransaction(segwit_tx_list[i]) != self.nodes[0].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i], 0) == self.nodes[2].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) != self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i]) == self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness())) + assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) + assert self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False) + + for tx_id in segwit_tx_list: + tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"]) + assert self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) @@ -195,11 +204,11 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data - assert(tmpl['weightlimit'] == 4000000) - assert(tmpl['sigoplimit'] == 80000) - assert(tmpl['transactions'][0]['txid'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 8) + assert tmpl['sizelimit'] >= 3999577 # actual maximum size is lower due to minimum mandatory non-witness data + assert tmpl['weightlimit'] == 4000000 + assert tmpl['sigoplimit'] == 80000 + assert tmpl['transactions'][0]['txid'] == txid + assert tmpl['transactions'][0]['sigops'] == 8 self.nodes[0].generate(1) # Mine a block to clear the gbt cache @@ -211,8 +220,8 @@ class SegWitTest(BitcoinTestFramework): txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) hex_tx = self.nodes[0].gettransaction(txid)['hex'] tx = FromHex(CTransaction(), hex_tx) - assert(tx.wit.is_null()) # This should not be a segwit input - assert(txid1 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() # This should not be a segwit input + assert txid1 in self.nodes[0].getrawmempool() # Now create tx2, which will spend from txid1. tx = CTransaction() @@ -221,7 +230,7 @@ class SegWitTest(BitcoinTestFramework): tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx))['hex'] txid2 = self.nodes[0].sendrawtransaction(tx2_hex) tx = FromHex(CTransaction(), tx2_hex) - assert(not tx.wit.is_null()) + assert not tx.wit.is_null() # Now create tx3, which will spend from txid2 tx = CTransaction() @@ -229,23 +238,15 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(int(49.95 * COIN), CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) # Huge fee tx.calc_sha256() txid3 = self.nodes[0].sendrawtransaction(ToHex(tx)) - assert(tx.wit.is_null()) - assert(txid3 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() + assert txid3 in self.nodes[0].getrawmempool() - # Now try calling getblocktemplate() without segwit support. - template = self.nodes[0].getblocktemplate() - - # Check that tx1 is the only transaction of the 3 in the template. - template_txids = [t['txid'] for t in template['transactions']] - assert(txid2 not in template_txids and txid3 not in template_txids) - assert(txid1 in template_txids) - - # Check that running with segwit support results in all 3 being included. + # Check that getblocktemplate includes all transactions. template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) template_txids = [t['txid'] for t in template['transactions']] - assert(txid1 in template_txids) - assert(txid2 in template_txids) - assert(txid3 in template_txids) + assert txid1 in template_txids + assert txid2 in template_txids + assert txid3 in template_txids # Check that wtxid is properly reported in mempool entry assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) @@ -397,22 +398,22 @@ class SegWitTest(BitcoinTestFramework): v = self.nodes[0].getaddressinfo(i) if (v['isscript']): bare = hex_str_to_bytes(v['hex']) - importlist.append(bytes_to_hex_str(bare)) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(bare)]))) + importlist.append(bare.hex()) + importlist.append(CScript([OP_0, sha256(bare)]).hex()) else: pubkey = hex_str_to_bytes(v['pubkey']) p2pk = CScript([pubkey, OP_CHECKSIG]) p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) - importlist.append(bytes_to_hex_str(p2pk)) - importlist.append(bytes_to_hex_str(p2pkh)) - importlist.append(bytes_to_hex_str(CScript([OP_0, hash160(pubkey)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pk)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pkh)]))) + importlist.append(p2pk.hex()) + importlist.append(p2pkh.hex()) + importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) - importlist.append(bytes_to_hex_str(unsolvablep2pkh)) - importlist.append(bytes_to_hex_str(unsolvablep2wshp2pkh)) - importlist.append(bytes_to_hex_str(op1)) - importlist.append(bytes_to_hex_str(p2wshop1)) + importlist.append(unsolvablep2pkh.hex()) + importlist.append(unsolvablep2wshp2pkh.hex()) + importlist.append(op1.hex()) + importlist.append(p2wshop1.hex()) for i in importlist: # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC @@ -540,10 +541,10 @@ class SegWitTest(BitcoinTestFramework): for i in script_list: tx.vout.append(CTxOut(10000000, i)) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - txid = self.nodes[0].sendrawtransaction(signresults, True) - self.nodes[0].generate(1) - sync_blocks(self.nodes) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + txid = self.nodes[0].sendrawtransaction(signresults, 0) + txs_mined[txid] = self.nodes[0].generate(1)[0] + self.sync_blocks() watchcount = 0 spendcount = 0 for i in self.nodes[0].listunspent(): @@ -585,17 +586,17 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() for i in txids: txtmp = CTransaction() - txraw = self.nodes[0].getrawtransaction(i) + txraw = self.nodes[0].getrawtransaction(i, 0, txs_mined[i]) f = BytesIO(hex_str_to_bytes(txraw)) txtmp.deserialize(f) for j in range(len(txtmp.vout)): tx.vin.append(CTxIn(COutPoint(int('0x' + i, 0), j))) tx.vout.append(CTxOut(0, CScript())) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] - self.nodes[0].sendrawtransaction(signresults, True) + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] + self.nodes[0].sendrawtransaction(signresults, 0) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() if __name__ == '__main__': diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py new file mode 100755 index 0000000000..5084cb1322 --- /dev/null +++ b/test/functional/feature_shutdown.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoind shutdown.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, get_rpc_proxy, wait_until +from threading import Thread + +def test_long_call(node): + block = node.waitfornewblock() + assert_equal(block['height'], 0) + +class ShutdownTest(BitcoinTestFramework): + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + node = get_rpc_proxy(self.nodes[0].url, 1, timeout=600, coveragedir=self.nodes[0].coverage_dir) + # Force connection establishment by executing a dummy command. + node.getblockcount() + Thread(target=test_long_call, args=(node,)).start() + # Wait until the server is executing the above `waitfornewblock`. + wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) + # Wait 1 second after requesting shutdown but not before the `stop` call + # finishes. This is to ensure event loop waits for current connections + # to close. + self.stop_node(0, wait=1000) + +if __name__ == '__main__': + ShutdownTest().main() diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 88df61cabc..0713925141 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -22,7 +22,6 @@ VB_TOP_BITS = 0x20000000 VB_UNKNOWN_BIT = 27 # Choose a bit unassigned to any deployment VB_UNKNOWN_VERSION = VB_TOP_BITS | (1 << VB_UNKNOWN_BIT) -WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible unknown rules are in effect" WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("Warning: unknown new rules activated.*versionbit") @@ -75,18 +74,13 @@ class VersionBitsWarningTest(BitcoinTestFramework): node.generatetoaddress(VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() - assert(not VB_PATTERN.match(node.getmininginfo()["warnings"])) - assert(not VB_PATTERN.match(node.getnetworkinfo()["warnings"])) + assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) + assert not VB_PATTERN.match(node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if >50 blocks in the last 100 were an unknown version") # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(node.p2p, VB_THRESHOLD, VB_UNKNOWN_VERSION) node.generatetoaddress(VB_PERIOD - VB_THRESHOLD, node_deterministic_address) - # Check that get*info() shows the 51/100 unknown block version error. - assert(WARN_UNKNOWN_RULES_MINED in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_MINED in node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if previous VB_BLOCKS have >=VB_THRESHOLD blocks with unknown versionbits version.") # Mine a period worth of expected blocks so the generic block-version warning # is cleared. This will move the versionbit state to ACTIVE. @@ -101,8 +95,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"]) + assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] + assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index aed439339f..15b7ba9ae1 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" @@ -16,7 +16,7 @@ class TestBitcoinCli(BitcoinTestFramework): """Main test logic""" cli_response = self.nodes[0].cli("-version").send_cli() - assert("Bitcoin Core RPC client version" in cli_response) + assert "Bitcoin Core RPC client version" in cli_response self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") if self.is_wallet_compiled(): @@ -57,16 +57,14 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) - if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) - assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) - assert_equal(cli_get_info['testnet'], blockchain_info['chain'] == "test") + assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): + assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index e4b86f9e1e..bb868d7115 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" @@ -30,14 +30,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -47,14 +47,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -64,8 +64,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock==None) #now the connection must be closed after the response + assert b'"error":null' in out1 + assert conn.sock is None #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -76,7 +76,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) + assert b'"error":null' in out1 #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urllib.parse.urlparse(self.nodes[2].url) @@ -87,8 +87,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock!=None) #connection must be closed because bitcoind should use keep-alive by default + assert b'"error":null' in out1 + assert conn.sock is not None #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index afa9de580f..a036dfc790 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" @@ -22,6 +22,8 @@ from test_framework.util import ( hex_str_to_bytes, ) +from test_framework.messages import BLOCK_HEADER_SIZE + class ReqType(Enum): JSON = 1 BIN = 2 @@ -88,15 +90,17 @@ class RESTTest (BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() - self.nodes[1].generatetoaddress(1, not_related_address) - self.sync_all() - bb_hash = self.nodes[0].getbestblockhash() - assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) - - self.log.info("Load the transaction using the /tx URI") + self.log.info("Test the /tx URI") json_obj = self.test_rest_request("/tx/{}".format(txid)) + assert_equal(json_obj['txid'], txid) + + # Check hex format response + hex_response = self.test_rest_request("/tx/{}".format(txid), req_type=ReqType.HEX, ret_type=RetType.OBJ) + assert_greater_than_or_equal(int(hex_response.getheader('content-length')), + json_obj['size']*2) + spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) # get the vin to later check for utxo (should be spent by then) # get n of 0.1 outpoint n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) @@ -104,9 +108,14 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query an unspent TXO using the /getutxos URI") - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + self.nodes[1].generatetoaddress(1, not_related_address) + self.sync_all() + bb_hash = self.nodes[0].getbestblockhash() + + assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) # Check chainTip response + json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) assert_equal(json_obj['chaintipHash'], bb_hash) # Make sure there is one utxo @@ -143,7 +152,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) chain_height, = unpack("i", output.read(4)) - response_hash = binascii.hexlify(output.read(32)[::-1]).decode('ascii') + response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine assert_equal(chain_height, 102) # chain height must be 102 @@ -198,35 +207,62 @@ class RESTTest (BitcoinTestFramework): self.nodes[0].generate(1) # generate block to not affect upcoming tests self.sync_all() - self.log.info("Test the /block and /headers URIs") + self.log.info("Test the /block, /blockhashbyheight and /headers URIs") bb_hash = self.nodes[0].getbestblockhash() + # Check result if block does not exists + assert_equal(self.test_rest_request('/headers/1/0000000000000000000000000000000000000000000000000000000000000000'), []) + self.test_rest_request('/block/0000000000000000000000000000000000000000000000000000000000000000', status=404, ret_type=RetType.OBJ) + + # Check result if block is not in the active chain + self.nodes[0].invalidateblock(bb_hash) + assert_equal(self.test_rest_request('/headers/1/{}'.format(bb_hash)), []) + self.test_rest_request('/block/{}'.format(bb_hash)) + self.nodes[0].reconsiderblock(bb_hash) + # Check binary format response = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) - assert_greater_than(int(response.getheader('content-length')), 80) + assert_greater_than(int(response.getheader('content-length')), BLOCK_HEADER_SIZE) response_bytes = response.read() # Compare with block header response_header = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) - assert_equal(int(response_header.getheader('content-length')), 80) + assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() - assert_equal(response_bytes[:80], response_header_bytes) + assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) # Check block hex format response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than(int(response_hex.getheader('content-length')), 160) + assert_greater_than(int(response_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_hex_bytes = response_hex.read().strip(b'\n') assert_equal(binascii.hexlify(response_bytes), response_hex_bytes) # Compare with hex block header response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than(int(response_header_hex.getheader('content-length')), 160) - response_header_hex_bytes = response_header_hex.read(160) - assert_equal(binascii.hexlify(response_bytes[:80]), response_header_hex_bytes) + assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) + response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) + assert_equal(binascii.hexlify(response_bytes[:BLOCK_HEADER_SIZE]), response_header_hex_bytes) # Check json format block_json_obj = self.test_rest_request("/block/{}".format(bb_hash)) assert_equal(block_json_obj['hash'], bb_hash) + assert_equal(self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']))['blockhash'], bb_hash) + + # Check hex/bin format + resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) + assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) + resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) + blockhash = resp_bytes[::-1].hex() + assert_equal(blockhash, bb_hash) + + # Check invalid blockhashbyheight requests + resp = self.test_rest_request("/blockhashbyheight/abc", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: abc") + resp = self.test_rest_request("/blockhashbyheight/1000000", ret_type=RetType.OBJ, status=404) + assert_equal(resp.read().decode('utf-8').rstrip(), "Block height out of range") + resp = self.test_rest_request("/blockhashbyheight/-1", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: -1") + self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header json_obj = self.test_rest_request("/headers/1/{}".format(bb_hash)) @@ -244,17 +280,6 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash)) assert_equal(len(json_obj), 5) # now we should have 5 header objects - self.log.info("Test the /tx URI") - - tx_hash = block_json_obj['tx'][0]['txid'] - json_obj = self.test_rest_request("/tx/{}".format(tx_hash)) - assert_equal(json_obj['txid'], tx_hash) - - # Check hex format response - hex_response = self.test_rest_request("/tx/{}".format(tx_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than_or_equal(int(hex_response.getheader('content-length')), - json_obj['size']*2) - self.log.info("Test tx inclusion in the /mempool and /block URIs") # Make 3 tx and mine them on node 1 diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index e3d7b0655d..49ae0fb1a9 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,17 +1,37 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import assert_equal, assert_greater_than_or_equal + +def expect_http_status(expected_http_status, expected_rpc_code, + fcn, *args): + try: + fcn(*args) + raise AssertionError("Expected RPC error %d, got none" % expected_rpc_code) + except JSONRPCException as exc: + assert_equal(exc.error["code"], expected_rpc_code) + assert_equal(exc.http_status, expected_http_status) class RPCInterfaceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + def test_getrpcinfo(self): + self.log.info("Testing getrpcinfo...") + + info = self.nodes[0].getrpcinfo() + assert_equal(len(info['active_commands']), 1) + + command = info['active_commands'][0] + assert_equal(command['method'], 'getrpcinfo') + assert_greater_than_or_equal(command['duration'], 0) + def test_batch_request(self): self.log.info("Testing basic JSON-RPC batch request...") @@ -38,8 +58,16 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_equal(result_by_id[3]['error'], None) assert result_by_id[3]['result'] is not None + def test_http_status_codes(self): + self.log.info("Testing HTTP status codes for JSON-RPC requests...") + + expect_http_status(404, -32601, self.nodes[0].invalidmethod) + expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + def run_test(self): + self.test_getrpcinfo() self.test_batch_request() + self.test_http_status_codes() if __name__ == '__main__': diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 94fea37090..8e58c85c15 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" @@ -10,7 +10,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction from test_framework.util import ( assert_equal, - bytes_to_hex_str, hash256, ) from io import BytesIO @@ -94,17 +93,17 @@ class ZMQTest (BitcoinTestFramework): tx = CTransaction() tx.deserialize(BytesIO(hex)) tx.calc_sha256() - assert_equal(tx.hash, bytes_to_hex_str(txid)) + assert_equal(tx.hash, txid.hex()) # Should receive the generated block hash. - hash = bytes_to_hex_str(self.hashblock.receive()) + hash = self.hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. - assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) # Should receive the generated raw block. block = self.rawblock.receive() - assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) + assert_equal(genhashes[x], hash256(block[:80]).hex()) if self.is_wallet_compiled(): self.log.info("Wait for tx from second node") @@ -113,11 +112,11 @@ class ZMQTest (BitcoinTestFramework): # Should receive the broadcasted txid. txid = self.hashtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(txid)) + assert_equal(payment_txid, txid.hex()) # Should receive the broadcasted raw transaction. hex = self.rawtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) + assert_equal(payment_txid, hash256(hex).hex()) self.log.info("Test the getzmqnotifications RPC") diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 8847777ba7..2bb5d8ab7d 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" from io import BytesIO +import math + from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, @@ -25,9 +27,7 @@ from test_framework.script import ( from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str, hex_str_to_bytes, - wait_until, ) @@ -36,7 +36,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ '-txindex', - '-reindex', # Need reindex for txindex '-acceptnonstdtxn=0', # Try to mimic main-net ]] * self.num_nodes @@ -54,8 +53,9 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 - wait_until(lambda: node.getblockcount() == 200) + assert_equal(node.getblockcount(), 200) assert_equal(node.getmempoolinfo()['size'], self.mempool_size) + coins = node.listunspent() self.log.info('Should not accept garbage to testmempoolaccept') assert_raises_rpc_error(-3, 'Expected type array, got string', lambda: node.testmempoolaccept(rawtxs='ff00baar')) @@ -63,13 +63,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): assert_raises_rpc_error(-22, 'TX decode failed', lambda: node.testmempoolaccept(rawtxs=['ff00baar'])) self.log.info('A transaction already in the blockchain') - coin = node.listunspent()[0] # Pick a random coin(base) to spend + coin = coins.pop() # Pick a random coin(base) to spend raw_tx_in_block = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[{'txid': coin['txid'], 'vout': coin['vout']}], outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], ))['hex'] - txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True) + txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, maxfeerate=0) node.generate(1) + self.mempool_size = 0 self.check_mempool_result( result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': '18: txn-already-known'}], rawtxs=[raw_tx_in_block], @@ -89,9 +90,25 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[raw_tx_0], ) + self.log.info('A final transaction not in the mempool') + coin = coins.pop() # Pick a random coin(base) to spend + raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction( + inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}], # SEQUENCE_FINAL + outputs=[{node.getnewaddress(): 0.025}], + locktime=node.getblockcount() + 2000, # Can be anything + ))['hex'] + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': True}], + rawtxs=[tx.serialize().hex()], + maxfeerate=0, + ) + node.sendrawtransaction(hexstring=raw_tx_final, maxfeerate=0) + self.mempool_size += 1 + self.log.info('A transaction in the mempool') node.sendrawtransaction(hexstring=raw_tx_0) - self.mempool_size = 1 + self.mempool_size += 1 self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': '18: txn-already-in-mempool'}], rawtxs=[raw_tx_0], @@ -101,7 +118,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(fee * COIN) # Double the fee tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER + 1 # Now, opt out of RBF - raw_tx_0 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] + raw_tx_0 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( @@ -111,15 +128,15 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that conflicts with an unconfirmed tx') # Send the transaction that replaces the mempool transaction and opts out of replaceability - node.sendrawtransaction(hexstring=bytes_to_hex_str(tx.serialize()), allowhighfees=True) + node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) # take original raw_tx_0 tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '18: txn-mempool-conflict'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) self.log.info('A transaction with missing inputs, that never existed') @@ -128,14 +145,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'missing-inputs'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with missing inputs, that existed once in the past') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vin[0].prevout.n = 1 # Set vout to 1, to spend the other outpoint (49 coins) of the in-chain-tx we want to double spend - raw_tx_1 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] - txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, allowhighfees=True) + raw_tx_1 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] + txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, maxfeerate=0) # Now spend both to "clearly hide" the outputs, ie. remove the coins from the utxo set by spending them raw_tx_spend_both = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[ @@ -144,7 +161,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ], outputs=[{node.getnewaddress(): 0.1}] ))['hex'] - txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, allowhighfees=True) + txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, maxfeerate=0) node.generate(1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again @@ -166,25 +183,25 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Reference tx should be valid on itself self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with no outputs') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout = [] # Skip re-signing the transaction for context independent checks from now on - # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex']))) + # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-empty'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A really large transaction') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) - tx.vin = [tx.vin[0]] * (MAX_BLOCK_BASE_SIZE // len(tx.vin[0].serialize())) + tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with negative output value') @@ -192,7 +209,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue *= -1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-negative'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large output value') @@ -200,7 +217,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large sum of output values') @@ -209,7 +226,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-txouttotal-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with duplicate inputs') @@ -217,7 +234,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-inputs-duplicate'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A coinbase transaction') @@ -226,7 +243,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_coinbase_spent))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: coinbase'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('Some nonstandard transactions') @@ -234,19 +251,19 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nVersion = 3 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: version'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_0]) # Some non-standard script self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptpubkey'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptsig-not-pushonly'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) @@ -254,21 +271,21 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: tx-size'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0] = output_p2sh_burn tx.vout[0].nValue -= 1 # Make output smaller, such that it is dust for our policy self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: dust'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'\xff']) tx.vout = [tx.vout[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: multi-op-return'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A timelocked transaction') @@ -277,7 +294,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nLockTime = node.getblockcount() + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction that is locked by BIP68 sequence logic') @@ -286,8 +303,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Can skip re-signing the tx because of early rejection self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-BIP68-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], - allowhighfees=True, + rawtxs=[tx.serialize().hex()], + maxfeerate=0, ) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index c0918893cd..351b27e94a 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" @@ -47,9 +47,9 @@ class MempoolLimitTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) self.log.info('The tx should be evicted by now') - assert(txid not in self.nodes[0].getrawmempool()) + assert txid not in self.nodes[0].getrawmempool() txdata = self.nodes[0].gettransaction(txid) - assert(txdata['confirmations'] == 0) #confirmation should still be 0 + assert txdata['confirmations'] == 0 #confirmation should still be 0 self.log.info('Check that mempoolminfee is larger than minrelytxfee') assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 9336547a6b..c7d241503a 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" @@ -8,7 +8,11 @@ from decimal import Decimal from test_framework.messages import COIN from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + satoshi_round, +) MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 @@ -33,7 +37,7 @@ class MempoolPackagesTest(BitcoinTestFramework): signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx['hex']) fulltx = node.getrawtransaction(txid, 1) - assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output + assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output return (txid, send_value) def run_test(self): @@ -58,9 +62,9 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(mempool), MAX_ANCESTORS) descendant_count = 1 descendant_fees = 0 - descendant_size = 0 + descendant_vsize = 0 - ancestor_size = sum([mempool[tx]['size'] for tx in mempool]) + ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool]) ancestor_count = MAX_ANCESTORS ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool]) @@ -79,15 +83,15 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_size += mempool[x]['size'] - assert_equal(mempool[x]['descendantsize'], descendant_size) + descendant_vsize += mempool[x]['vsize'] + assert_equal(mempool[x]['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct assert_equal(mempool[x]['ancestorcount'], ancestor_count) assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_size) - ancestor_size -= mempool[x]['size'] + assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) + ancestor_vsize -= mempool[x]['vsize'] ancestor_fees -= mempool[x]['fee'] ancestor_count -= 1 @@ -125,13 +129,13 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(v_ancestors), len(chain)-1) for x in v_ancestors.keys(): assert_equal(mempool[x], v_ancestors[x]) - assert(chain[-1] not in v_ancestors.keys()) + assert chain[-1] not in v_ancestors.keys() v_descendants = self.nodes[0].getmempooldescendants(chain[0], True) assert_equal(len(v_descendants), len(chain)-1) for x in v_descendants.keys(): assert_equal(mempool[x], v_descendants[x]) - assert(chain[0] not in v_descendants.keys()) + assert chain[0] not in v_descendants.keys() # Check that ancestor modified fees includes fee deltas from # prioritisetransaction @@ -163,7 +167,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) # Prioritise a transaction that has been mined, then add it back to the # mempool by using invalidateblock. @@ -228,7 +232,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Test reorg handling # First, the basics: self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash()) self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash()) @@ -283,12 +287,12 @@ class MempoolPackagesTest(BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction(inputs, outputs) signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) txid = self.nodes[0].sendrawtransaction(signedtx['hex']) - sync_mempools(self.nodes) + self.sync_mempools() # Now try to disconnect the tip on each node... self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - sync_blocks(self.nodes) + self.sync_blocks() if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index b4e9d967fd..d74d4eaaf1 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -42,6 +42,7 @@ import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, wait_until + class MempoolPersistTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 @@ -60,7 +61,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Send 5 transactions from node2 (to its own address)") for i in range(5): - self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) node2_balance = self.nodes[2].getbalance() self.sync_all() @@ -68,6 +69,13 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + self.log.debug("Prioritize a transaction on node0") + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'], fees['modified']) + self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000) + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -81,6 +89,10 @@ class MempoolPersistTest(BitcoinTestFramework): # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) + self.log.debug('Verify prioritization is loaded correctly') + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + # Verify accounting of mempool transactions after restart is correct self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 845beb551e..187c9026f6 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" @@ -45,7 +45,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 # Use invalidateblock to re-org back for node in self.nodes: @@ -55,7 +55,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] == 0) + assert tx["confirmations"] == 0 # Generate another block, they should all get mined self.nodes[0].generate(1) @@ -63,7 +63,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 if __name__ == '__main__': diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 9f01be0646..788aabc192 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs @@ -11,10 +11,14 @@ import copy from decimal import Decimal -from test_framework.blocktools import create_coinbase +from test_framework.blocktools import ( + create_coinbase, + TIME_GENESIS_BLOCK, +) from test_framework.messages import ( CBlock, CBlockHeader, + BLOCK_HEADER_SIZE ) from test_framework.mininode import ( P2PDataStore, @@ -23,53 +27,75 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str as b2x, + connect_nodes_bi, ) +from test_framework.script import CScriptNum def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() - rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal'}) + rsp = node.getblocktemplate(template_request={'data': block.serialize().hex(), 'mode': 'proposal', 'rules': ['segwit']}) assert_equal(rsp, expect) class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = False + self.setup_clean_chain = True + + def mine_chain(self): + self.log.info('Create some old blocks') + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + self.nodes[0].setmocktime(t) + self.nodes[0].generate(1) + mining_info = self.nodes[0].getmininginfo() + assert_equal(mining_info['blocks'], 200) + assert_equal(mining_info['currentblocktx'], 0) + assert_equal(mining_info['currentblockweight'], 4000) + self.restart_node(0) + connect_nodes_bi(self.nodes, 0, 1) def run_test(self): + self.mine_chain() node = self.nodes[0] def assert_submitblock(block, result_str_1, result_str_2=None): block.solve() result_str_2 = result_str_2 or 'duplicate-invalid' - assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize()))) - assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize()))) + assert_equal(result_str_1, node.submitblock(hexdata=block.serialize().hex())) + assert_equal(result_str_2, node.submitblock(hexdata=block.serialize().hex())) self.log.info('getmininginfo') mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['chain'], 'regtest') - assert_equal(mining_info['currentblocktx'], 0) - assert_equal(mining_info['currentblockweight'], 0) + assert 'currentblocktx' not in mining_info + assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334')) assert_equal(mining_info['pooledtx'], 0) # Mine a block to leave initial block download node.generatetoaddress(1, node.get_deterministic_priv_key().address) - tmpl = node.getblocktemplate() + tmpl = node.getblocktemplate({'rules': ['segwit']}) self.log.info("getblocktemplate: Test capability advertised") assert 'proposal' in tmpl['capabilities'] assert 'coinbasetxn' not in tmpl - coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1) + next_height = int(tmpl["height"]) + coinbase_tx = create_coinbase(height=next_height) # sequence numbers must not be max for nLockTime to have effect coinbase_tx.vin[0].nSequence = 2 ** 32 - 2 coinbase_tx.rehash() + # round-trip the encoded bip34 block height commitment + assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height) + # round-trip negative and multi-byte CScriptNums to catch python regression + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500) + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500) + assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1) + block = CBlock() block.nVersion = tmpl["version"] block.hashPrevBlock = int(tmpl["previousblockhash"], 16) @@ -78,11 +104,14 @@ class MiningTest(BitcoinTestFramework): block.nNonce = 0 block.vtx = [coinbase_tx] + self.log.info("getblocktemplate: segwit rule must be set") + assert_raises_rpc_error(-8, "getblocktemplate must be called with the segwit rule set", node.getblocktemplate) + self.log.info("getblocktemplate: Test valid block") assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, block.serialize()[:-15].hex()) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -91,10 +120,10 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': block.serialize()[:-1].hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -120,11 +149,10 @@ class MiningTest(BitcoinTestFramework): self.log.info("getblocktemplate: Test bad tx count") # The tx count is immediately after the block header - TX_COUNT_OFFSET = 80 bad_block_sn = bytearray(block.serialize()) - assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) - bad_block_sn[TX_COUNT_OFFSET] += 1 - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1) + bad_block_sn[BLOCK_HEADER_SIZE] += 1 + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': bad_block_sn.hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) @@ -153,9 +181,9 @@ class MiningTest(BitcoinTestFramework): assert_submitblock(bad_block, 'prev-blk-not-found', 'prev-blk-not-found') self.log.info('submitheader tests') - assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * 80)) - assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * 78)) - assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata='ff' * 80)) + assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE)) + assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2))) + assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=super(CBlock, bad_block).serialize().hex())) block.nTime += 1 block.solve() @@ -164,23 +192,23 @@ class MiningTest(BitcoinTestFramework): return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status} assert chain_tip(block.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(block.serialize())) + node.submitheader(hexdata=block.serialize().hex()) assert chain_tip(block.hash) in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) # Noop + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) # Noop assert chain_tip(block.hash) in node.getchaintips() bad_block_root = copy.deepcopy(block) bad_block_root.hashMerkleRoot += 2 bad_block_root.solve() assert chain_tip(bad_block_root.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() # Should still reject invalid blocks, even if we have the header: - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') assert chain_tip(bad_block_root.hash) in node.getchaintips() # We know the header for this invalid block, so should just return early without error: - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() bad_block_lock = copy.deepcopy(block) @@ -188,19 +216,19 @@ class MiningTest(BitcoinTestFramework): bad_block_lock.vtx[0].rehash() bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root() bad_block_lock.solve() - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal') - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'bad-txns-nonfinal') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'duplicate-invalid') # Build a "good" block on top of the submitted bad block bad_block2 = copy.deepcopy(block) bad_block2.hashPrevBlock = bad_block_lock.sha256 bad_block2.solve() - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) # Should reject invalid header right away bad_block_time = copy.deepcopy(block) bad_block_time.nTime = 1 bad_block_time.solve() - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) # Should ask for the block from a p2p node, if they announce the header as well: node.add_p2p_connection(P2PDataStore()) @@ -211,11 +239,11 @@ class MiningTest(BitcoinTestFramework): # Building a few blocks should give the same results node.generatetoaddress(10, node.get_deterministic_priv_key().address) - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) - assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate') # valid + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) + assert_equal(node.submitblock(hexdata=block.serialize().hex()), 'duplicate') # valid if __name__ == '__main__': diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 9a3c15a4a7..445ec124ce 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" @@ -15,14 +15,14 @@ class LongpollThread(threading.Thread): def __init__(self, node): threading.Thread.__init__(self) # query current longpollid - template = node.getblocktemplate() + template = node.getblocktemplate({'rules': ['segwit']}) self.longpollid = template['longpollid'] # create a new connection to the node, we can't use the same # connection from two threads self.node = get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) def run(self): - self.node.getblocktemplate({'longpollid':self.longpollid}) + self.node.getblocktemplate({'longpollid': self.longpollid, 'rules': ['segwit']}) class GetBlockTemplateLPTest(BitcoinTestFramework): def set_test_params(self): @@ -34,31 +34,31 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): def run_test(self): self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.") self.nodes[0].generate(10) - template = self.nodes[0].getblocktemplate() + template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) longpollid = template['longpollid'] # longpollid should not change between successive invocations if nothing else happens - template2 = self.nodes[0].getblocktemplate() - assert(template2['longpollid'] == longpollid) + template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) + assert template2['longpollid'] == longpollid # Test 1: test that the longpolling wait if we do nothing thr = LongpollThread(self.nodes[0]) thr.start() # check that thread still lives thr.join(5) # wait 5 seconds or until thread exits - assert(thr.is_alive()) + assert thr.is_alive() # Test 2: test that longpoll will terminate if another node generates a block self.nodes[1].generate(1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 3: test that longpoll will terminate if we generate a block ourselves thr = LongpollThread(self.nodes[0]) thr.start() self.nodes[0].generate(1) # generate a block on another node thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll thr = LongpollThread(self.nodes[0]) @@ -69,7 +69,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) - assert(not thr.is_alive()) + assert not thr.is_alive() if __name__ == '__main__': GetBlockTemplateLPTest().main() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index c5ddee56f1..b0a069be81 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -63,9 +63,9 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes = [0, 0, 0] for i in range(3): for j in txids[i]: - assert(j in mempool) - sizes[i] += mempool[j]['size'] - assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count + assert j in mempool + sizes[i] += mempool[j]['vsize'] + assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined @@ -75,8 +75,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that prioritised transaction was mined") - assert(txids[0][0] not in mempool) - assert(txids[0][1] in mempool) + assert txids[0][0] not in mempool + assert txids[0][1] in mempool high_fee_tx = None for x in txids[2]: @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx != None) + assert high_fee_tx is not None # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). @@ -95,7 +95,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Check to make sure our high fee rate tx is back in the mempool mempool = self.nodes[0].getrawmempool() - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool # Now verify the modified-high feerate transaction isn't mined before # the other high fee transactions. Keep mining until our mempool has @@ -107,14 +107,14 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # transactions should have been. mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool for x in txids[2]: if (x != high_fee_tx): - assert(x not in mempool) + assert x not in mempool # Create a free transaction. Should be rejected. utxo_list = self.nodes[0].listunspent() - assert(len(utxo_list) > 0) + assert len(utxo_list) > 0 utxo = utxo_list[0] inputs = [] @@ -127,7 +127,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # This will raise an exception due to min relay fee not being met assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) - assert(tx_id not in self.nodes[0].getrawmempool()) + assert tx_id not in self.nodes[0].getrawmempool() # This is a less than 1000-byte transaction, so just set the fee # to be the minimum for a 1000-byte transaction and check that it is @@ -136,18 +136,18 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) - assert(tx_id in self.nodes[0].getrawmempool()) + assert tx_id in self.nodes[0].getrawmempool() # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. mock_time = int(time.time()) self.nodes[0].setmocktime(mock_time) - template = self.nodes[0].getblocktemplate() + template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN)) self.nodes[0].setmocktime(mock_time+10) - new_template = self.nodes[0].getblocktemplate() + new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template != new_template) + assert template != new_template if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index f0dee59b5c..3ca6bec254 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -7,19 +7,18 @@ Version 1 compact blocks are pre-segwit (txids) Version 2 compact blocks are post-segwit (wtxids) """ -from decimal import Decimal import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_witness_block, msg_witness_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_bip9_status, satoshi_round, sync_blocks, wait_until +from test_framework.util import assert_equal, get_bip9_status, wait_until # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self): + def __init__(self, cmpct_version): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -27,6 +26,7 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() + self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -94,11 +94,7 @@ class TestP2PConn(P2PInterface): class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] + self.num_nodes = 1 self.utxos = [] def skip_test_if_missing_module(self): @@ -117,11 +113,10 @@ class CompactBlocksTest(BitcoinTestFramework): # Create 10 more anyone-can-spend utxo's for testing. def make_utxos(self): - # Doesn't matter which node we use, just use node0. block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) - self.nodes[0].generate(100) + self.segwit_node.send_and_ping(msg_block(block)) + assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 + self.nodes[0].generatetoaddress(100, self.nodes[0].getnewaddress(address_type="bech32")) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -135,10 +130,10 @@ class CompactBlocksTest(BitcoinTestFramework): block2.vtx.append(tx) block2.hashMerkleRoot = block2.calc_merkle_root() block2.solve() - self.test_node.send_and_ping(msg_block(block2)) + self.segwit_node.send_and_ping(msg_block(block2)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return + # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. @@ -149,7 +144,10 @@ class CompactBlocksTest(BitcoinTestFramework): # are made with compact blocks. # If old_node is passed in, request compact blocks with version=preferred-1 # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + def test_sendcmpct(self, test_node, old_node=None): + preferred_version = test_node.cmpct_version + node = self.nodes[0] + # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) @@ -167,7 +165,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.clear_block_announcement() block_hash = int(node.generate(1)[0], 16) peer.wait_for_block_announcement(block_hash, timeout=30) - assert(peer.block_announced) + assert peer.block_announced with mininode_lock: assert predicate(peer), ( @@ -251,23 +249,18 @@ class CompactBlocksTest(BitcoinTestFramework): # This index will be too high prefilled_txn = PrefilledTransaction(1, block.vtx[0]) cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): + def test_compactblock_construction(self, test_node, use_witness_address=True): + version = test_node.cmpct_version + node = self.nodes[0] # Generate a bunch of transactions. node.generate(101) num_transactions = 25 address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.getnewaddress(address_type='bech32') - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send - Decimal(0.1))) - node.generate(1) segwit_tx_generated = False for i in range(num_transactions): @@ -285,7 +278,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.wait_for_block_announcement(tip) # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) + self.request_cb_announcements(test_node) # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() @@ -303,7 +296,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -319,7 +312,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -330,7 +323,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(header_and_shortids.header.sha256, block_hash) # Make sure the prefilled_txn appears to have included the coinbase - assert(len(header_and_shortids.prefilled_txn) >= 1) + assert len(header_and_shortids.prefilled_txn) >= 1 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) # Check that all prefilled_txn entries match what's in the block. @@ -345,7 +338,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) else: # Shouldn't have received a witness - assert(entry.tx.wit.is_null()) + assert entry.tx.wit.is_null() # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -375,7 +368,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Post-segwit: upgraded nodes would only make this request of cb-version-2, # NODE_WITNESS peers. Unupgraded nodes would still make this request of # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): + def test_compactblock_requests(self, test_node, segwit=True): + version = test_node.cmpct_version + node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: @@ -407,7 +402,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -440,14 +435,16 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we only receive getblocktxn requests for transactions that the # node needs, and that responding to them causes the block to be # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): + def test_getblocktxn_requests(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert("getblocktxn" in peer.last_message) + assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -487,7 +484,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in node.getrawmempool()) + assert block.vtx[1].hash in node.getrawmempool() # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. @@ -508,7 +505,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Clear out last request. with mininode_lock: @@ -519,13 +516,13 @@ class CompactBlocksTest(BitcoinTestFramework): test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert("getblocktxn" not in test_node.last_message) + assert "getblocktxn" not in test_node.last_message # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() + def test_incorrect_blocktxn_response(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -537,7 +534,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:6]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Send compact block comp_block = HeaderAndShortIDs() @@ -545,7 +542,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -569,7 +566,7 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG) + assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block @@ -579,7 +576,9 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) - def test_getblocktxn_handler(self, node, test_node, version): + def test_getblocktxn_handler(self, test_node): + version = test_node.cmpct_version + node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. MAX_GETBLOCKTXN_DEPTH = 10 @@ -606,7 +605,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: # Witnesses should have been stripped - assert(tx.wit.is_null()) + assert tx.wit.is_null() else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) @@ -626,7 +625,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message - def test_compactblocks_not_at_tip(self, node, test_node): + def test_compactblocks_not_at_tip(self, test_node): + node = self.nodes[0] # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] @@ -669,7 +669,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(x["status"], "headers-only") found = True break - assert(found) + assert found # Requesting this block via getblocktxn should silently fail # (to avoid fingerprinting attacks). @@ -681,11 +681,8 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert "blocktxn" not in test_node.last_message - def activate_segwit(self, node): - node.generate(144 * 3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') - - def test_end_to_end_block_relay(self, node, listeners): + def test_end_to_end_block_relay(self, listeners): + node = self.nodes[0] utxo = self.utxos.pop(0) block = self.build_block_with_transactions(node, utxo, 10) @@ -706,8 +703,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert(len(self.utxos)) + def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + node = self.nodes[0] + assert len(self.utxos) utxo = self.utxos[0] block = self.build_block_with_transactions(node, utxo, 5) @@ -728,22 +726,24 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg) # Check that the tip didn't advance - assert(int(node.getbestblockhash(), 16) is not block.sha256) + assert int(node.getbestblockhash(), 16) is not block.sha256 test_node.sync_with_ping() # Helper for enabling cb announcements # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): + def request_cb_announcements(self, peer): + node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) msg = msg_sendcmpct() - msg.version = version + msg.version = peer.cmpct_version msg.announce = True peer.send_and_ping(msg) - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert(len(self.utxos)) + def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + node = self.nodes[0] + assert len(self.utxos) def announce_cmpct_block(node, peer): utxo = self.utxos.pop(0) @@ -764,7 +764,7 @@ class CompactBlocksTest(BitcoinTestFramework): delivery_peer.sync_with_ping() mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.sha256) @@ -783,7 +783,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.use_witness = True delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert(int(node.getbestblockhash(), 16) != block.sha256) + assert int(node.getbestblockhash(), 16) != block.sha256 msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 @@ -793,126 +793,55 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) # We will need UTXOs to construct transactions in later tests. self.make_utxos() - self.log.info("Running tests, pre-segwit activation:") + assert_equal(get_bip9_status(self.nodes[0], "segwit")["status"], 'active') self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) + self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) - - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + self.test_compactblock_construction(self.old_node) + self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) + self.test_compactblock_requests(self.segwit_node) self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.segwit_node) self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + self.test_getblocktxn_handler(self.segwit_node) + self.test_getblocktxn_handler(self.old_node) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.segwit_node) + self.test_compactblocks_not_at_tip(self.old_node) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.segwit_node) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.segwit_node, self.additional_segwit_node) # Test that if we submitblock to node1, we'll get a compact block # announcement to all peers. # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + self.request_cb_announcements(self.old_node) + self.request_cb_announcements(self.segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + self.test_invalid_tx_in_compactblock(self.segwit_node) + self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index d589519e45..7f901b1886 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" @@ -10,7 +10,7 @@ import time from test_framework.messages import msg_feefilter from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import sync_blocks, sync_mempools + def hashToHex(hash): return format(hash, '064x') @@ -50,14 +50,14 @@ class FeeFilterTest(BitcoinTestFramework): node0 = self.nodes[0] # Get out of IBD node1.generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[0].add_p2p_connection(TestP2PConn()) # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte @@ -65,13 +65,13 @@ class FeeFilterTest(BitcoinTestFramework): # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - sync_mempools(self.nodes) # must be sure node 0 has received all txs + self.sync_mempools() # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we # we can sync the test on receipt (if node1's txs were relayed, they'd @@ -82,13 +82,13 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() if __name__ == '__main__': diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index c8c752d1f7..33b7060060 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers +# Copyright (c) 2015-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid locators. diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 3ee67980a4..481d697e63 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" +import asyncio +import os import struct from test_framework import messages -from test_framework.mininode import P2PDataStore +from test_framework.mininode import P2PDataStore, NetworkThread from test_framework.test_framework import BitcoinTestFramework @@ -15,7 +17,7 @@ class msg_unrecognized: command = b'badmsg' - def __init__(self, str_data): + def __init__(self, *, str_data): self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data def serialize(self): @@ -25,19 +27,14 @@ class msg_unrecognized: return "{}(data={})".format(self.command, self.str_data) -class msg_nametoolong(msg_unrecognized): - - command = b'thisnameiswayyyyyyyyytoolong' - - class InvalidMessagesTest(BitcoinTestFramework): - def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True def run_test(self): """ + . Test msg header 0. Send a bunch of large (4MB) messages of an unrecognized type. Check to see that it isn't an effective DoS against the node. @@ -45,10 +42,12 @@ class InvalidMessagesTest(BitcoinTestFramework): 2. Send a few messages with an incorrect data size in the header, ensure the messages are ignored. - - 3. Send an unrecognized message with a command name longer than 12 characters. - """ + self.test_magic_bytes() + self.test_checksum() + self.test_size() + self.test_command() + node = self.nodes[0] self.node = node node.add_p2p_connection(P2PDataStore()) @@ -63,10 +62,13 @@ class InvalidMessagesTest(BitcoinTestFramework): # Send as large a message as is valid, ensure we aren't disconnected but # also can't exhaust resources. # - msg_at_size = msg_unrecognized("b" * valid_data_limit) + msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) assert len(msg_at_size.serialize()) == msg_limit - with node.assert_memory_usage_stable(increase_allowed=0.5): + increase_allowed = 0.5 + if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]: + increase_allowed = 3.5 + with node.assert_memory_usage_stable(increase_allowed=increase_allowed): self.log.info( "Sending a bunch of large, junk messages to test " "memory exhaustion. May take a bit...") @@ -82,7 +84,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # Peer 1, despite serving up a bunch of nonsense, should still be connected. self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=30) + node.p2p.sync_with_ping(timeout=120) assert node.p2p.is_connected # @@ -90,10 +92,10 @@ class InvalidMessagesTest(BitcoinTestFramework): # # Send an oversized message, ensure we're disconnected. # - msg_over_size = msg_unrecognized("b" * (valid_data_limit + 1)) + msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1)) assert len(msg_over_size.serialize()) == (msg_limit + 1) - with node.assert_debug_log(["Oversized message from peer=0, disconnecting"]): + with node.assert_debug_log(["Oversized message from peer=4, disconnecting"]): # An unknown message type (or *any* message type) over # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. node.p2p.send_message(msg_over_size) @@ -109,7 +111,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # Send messages with an incorrect data size in the header. # actual_size = 100 - msg = msg_unrecognized("b" * actual_size) + msg = msg_unrecognized(str_data="b" * actual_size) # TODO: handle larger-than cases. I haven't been able to pin down what behavior to expect. for wrong_size in (2, 77, 78, 79): @@ -136,18 +138,66 @@ class InvalidMessagesTest(BitcoinTestFramework): node.disconnect_p2ps() node.add_p2p_connection(P2PDataStore()) - # - # 3. - # - # Send a message with a too-long command name. - # - node.p2p.send_message(msg_nametoolong("foobar")) - node.p2p.wait_for_disconnect(timeout=4) - # Node is still up. conn = node.add_p2p_connection(P2PDataStore()) conn.sync_with_ping() + def test_magic_bytes(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + + def swap_magic_bytes(): + conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes + conn.magic_bytes = b'\x00\x11\x22\x32' + + # Call .result() to block until the atomic swap is complete, otherwise + # we might run into races later on + asyncio.run_coroutine_threadsafe(asyncio.coroutine(swap_magic_bytes)(), NetworkThread.network_event_loop).result() + + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): + conn.send_message(messages.msg_ping(nonce=0xff)) + conn.wait_for_disconnect(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_checksum(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['ProcessMessages(badmsg, 2 bytes): CHECKSUM ERROR expected 78df0a04 was ffffffff']): + msg = conn.build_message(msg_unrecognized(str_data="d")) + cut_len = ( + 4 + # magic + 12 + # command + 4 #len + ) + # modify checksum + msg = msg[:cut_len] + b'\xff' * 4 + msg[cut_len + 4:] + self.nodes[0].p2p.send_raw_message(msg) + conn.sync_with_ping(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_size(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['']): + msg = conn.build_message(msg_unrecognized(str_data="d")) + cut_len = ( + 4 + # magic + 12 # command + ) + # modify len to MAX_SIZE + 1 + msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] + self.nodes[0].p2p.send_raw_message(msg) + conn.wait_for_disconnect(timeout=1) + self.nodes[0].disconnect_p2ps() + + def test_command(self): + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): + msg = msg_unrecognized(str_data="d") + msg.command = b'\xff' * 12 + msg = conn.build_message(msg) + # Modify command + msg = msg[:7] + b'\x00' + msg[7 + 1:] + self.nodes[0].p2p.send_raw_message(msg) + conn.sync_with_ping(timeout=1) + self.nodes[0].disconnect_p2ps() def _tweak_msg_data_size(self, message, wrong_size): """ @@ -170,6 +220,5 @@ class InvalidMessagesTest(BitcoinTestFramework): return raw_msg_with_wrong_size - if __name__ == '__main__': InvalidMessagesTest().main() diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 58e129b57d..1b18dd3e58 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -5,7 +5,7 @@ """Test node responses to invalid transactions. In this test we connect to one node over p2p, and test tx requests.""" -from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script +from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ( COIN, COutPoint, @@ -19,6 +19,7 @@ from test_framework.util import ( assert_equal, wait_until, ) +from data import invalid_txs class InvalidTxRequestTest(BitcoinTestFramework): @@ -63,12 +64,21 @@ class InvalidTxRequestTest(BitcoinTestFramework): self.log.info("Mature the block.") self.nodes[0].generatetoaddress(100, self.nodes[0].get_deterministic_priv_key().address) - # b'\x64' is OP_NOTIF - # Transaction will be rejected with code 16 (REJECT_INVALID) - # and we get disconnected immediately - self.log.info('Test a transaction that is rejected') - tx1 = create_tx_with_script(block1.vtx[0], 0, script_sig=b'\x64' * 35, amount=50 * COIN - 12000) - node.p2p.send_txs_and_test([tx1], node, success=False, expect_disconnect=True) + # Iterate through a list of known invalid transaction types, ensuring each is + # rejected. Some are consensus invalid and some just violate policy. + for BadTxTemplate in invalid_txs.iter_all_templates(): + self.log.info("Testing invalid transaction: %s", BadTxTemplate.__name__) + template = BadTxTemplate(spend_block=block1) + tx = template.get_tx() + node.p2p.send_txs_and_test( + [tx], node, success=False, + expect_disconnect=template.expect_disconnect, + reject_reason=template.reject_reason, + ) + + if template.expect_disconnect: + self.log.info("Reconnecting to peer") + self.reconnect_p2p() # Make two p2p connections to provide the node with orphans # * p2ps[0] will send valid orphan txs (one with low fee) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 336d34a81d..06049db54c 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. @@ -117,9 +117,9 @@ class P2PLeakTest(BitcoinTestFramework): wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) # Make sure no unexpected messages came in - assert(no_version_bannode.unexpected_msg == False) - assert(no_version_idlenode.unexpected_msg == False) - assert(no_verack_idlenode.unexpected_msg == False) + assert no_version_bannode.unexpected_msg == False + assert no_version_idlenode.unexpected_msg == False + assert no_verack_idlenode.unexpected_msg == False if __name__ == '__main__': diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 359880506e..573d5f5a5f 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_NETWORK_LIMITED. @@ -11,7 +11,13 @@ and that it responds to getdata requests for blocks correctly: from test_framework.messages import CInv, msg_getdata, msg_verack, NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, disconnect_nodes, connect_nodes_bi, sync_blocks, wait_until +from test_framework.util import ( + assert_equal, + disconnect_nodes, + connect_nodes_bi, + wait_until, +) + class P2PIgnoreInv(P2PInterface): firstAddrnServices = 0 @@ -60,7 +66,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") connect_nodes_bi(self.nodes, 0, 1) blocks = self.nodes[1].generatetoaddress(292, self.nodes[1].get_deterministic_priv_key().address) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) self.log.info("Make sure we can max retrieve block at tip-288.") node.send_getdata_for_block(blocks[1]) # last block in valid range @@ -86,7 +92,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): # because node 2 is in IBD and node 0 is a NODE_NETWORK_LIMITED peer, sync must not be possible connect_nodes_bi(self.nodes, 0, 2) try: - sync_blocks([self.nodes[0], self.nodes[2]], timeout=5) + self.sync_blocks([self.nodes[0], self.nodes[2]], timeout=5) except: pass # node2 must remain at height 0 @@ -96,7 +102,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 1, 2) # sync must be possible - sync_blocks(self.nodes) + self.sync_blocks() # disconnect all peers self.disconnect_all() @@ -108,7 +114,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) # sync must be possible, node 1 is no longer in IBD and should therefore connect to node 0 (NODE_NETWORK_LIMITED) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) if __name__ == '__main__': NodeNetworkLimitedTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index afbbfa8992..0a53ceee86 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" -from binascii import hexlify import math import random import struct import time from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER -from test_framework.key import CECKey, CPubKey +from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, CBlock, @@ -74,13 +73,10 @@ from test_framework.script import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, connect_nodes, disconnect_nodes, get_bip9_status, hex_str_to_bytes, - sync_blocks, - sync_mempools, ) # The versionbit bit used to signal activation of SegWit @@ -104,7 +100,7 @@ def get_p2pkh_script(pubkeyhash): def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): """Add signature for a P2PK witness program.""" tx_hash = SegwitVersion1SignatureHash(script, tx_to, in_idx, hashtype, value) - signature = key.sign(tx_hash) + chr(hashtype).encode('latin-1') + signature = key.sign_ecdsa(tx_hash) + chr(hashtype).encode('latin-1') tx_to.wit.vtxinwit[in_idx].scriptWitness.stack = [signature, script] tx_to.rehash() @@ -285,7 +281,7 @@ class SegWitTest(BitcoinTestFramework): func(self, *args, **kwargs) # Each subtest should leave some utxos for the next subtest assert self.utxo - sync_blocks(self.nodes) + self.sync_blocks() # Assert segwit status is as expected at end of subtest assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], self.segwit_status) @@ -317,7 +313,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_message(msg_witness_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed - assert(tx.hash in self.nodes[0].getrawmempool()) + assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) @@ -335,7 +331,7 @@ class SegWitTest(BitcoinTestFramework): # Verify the hash with witness differs from the txid # (otherwise our testing framework must be broken!) tx.rehash() - assert(tx.sha256 != tx.calc_sha256(with_witness=True)) + assert tx.sha256 != tx.calc_sha256(with_witness=True) # Construct a segwit-signaling block that includes the transaction. block = self.build_next_block(version=(VB_TOP_BITS | (1 << VB_WITNESS_BIT))) @@ -371,20 +367,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) block2 = self.build_next_block(version=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block2, True) block3 = self.build_next_block(version=(VB_TOP_BITS | (1 << 15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -413,8 +409,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, []) # This gives us a witness commitment. - assert(len(block.vtx[0].wit.vtxinwit) == 1) - assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) + assert len(block.vtx[0].wit.vtxinwit) == 1 + assert len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1 test_witness_block(self.nodes[0], self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) @@ -448,7 +444,7 @@ class SegWitTest(BitcoinTestFramework): msg.headers = [CBlockHeader(block4)] self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) - assert(block4.sha256 not in self.old_node.getdataset) + assert block4.sha256 not in self.old_node.getdataset @subtest def test_v0_outputs_arent_spendable(self): @@ -537,7 +533,7 @@ class SegWitTest(BitcoinTestFramework): """Mine enough blocks for segwit's vb state to be 'started'.""" height = self.nodes[0].getblockcount() # Will need to rewrite the tests here if we are past the first period - assert(height < VB_PERIOD - 1) + assert height < VB_PERIOD - 1 # Advance to end of period, status should now be 'started' self.nodes[0].generate(VB_PERIOD - height - 1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') @@ -545,50 +541,28 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_getblocktemplate_before_lockin(self): - # Node0 is segwit aware, node2 is not. - for node in [self.nodes[0], self.nodes[2]]: - gbt_results = node.getblocktemplate() - block_version = gbt_results['version'] - # If we're not indicating segwit support, we will still be - # signalling for segwit activation. - assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0]) - # If we don't specify the segwit rule, then we won't get a default - # commitment. - assert('default_witness_commitment' not in gbt_results) - - # Workaround: - # Can either change the tip, or change the mempool and wait 5 seconds - # to trigger a recomputation of getblocktemplate. txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) - # Using mocktime lets us avoid sleep() - sync_mempools(self.nodes) - self.nodes[0].setmocktime(int(time.time()) + 10) - self.nodes[2].setmocktime(int(time.time()) + 10) for node in [self.nodes[0], self.nodes[2]]: gbt_results = node.getblocktemplate({"rules": ["segwit"]}) block_version = gbt_results['version'] if node == self.nodes[2]: - # If this is a non-segwit node, we should still not get a witness + # If this is a non-segwit node, we should not get a witness # commitment, nor a version bit signalling segwit. assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) - assert('default_witness_commitment' not in gbt_results) + assert 'default_witness_commitment' not in gbt_results else: # For segwit-aware nodes, check the version bit and the witness # commitment are correct. - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - assert('default_witness_commitment' in gbt_results) + assert block_version & (1 << VB_WITNESS_BIT) != 0 + assert 'default_witness_commitment' in gbt_results witness_commitment = gbt_results['default_witness_commitment'] # Check that default_witness_commitment is present. witness_root = CBlock.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, bytes_to_hex_str(script)) - - # undo mocktime - self.nodes[0].setmocktime(0) - self.nodes[2].setmocktime(0) + assert_equal(witness_commitment, script.hex()) @subtest def advance_to_segwit_lockin(self): @@ -597,7 +571,7 @@ class SegWitTest(BitcoinTestFramework): # Advance to end of period, and verify lock-in happens at the end self.nodes[0].generate(VB_PERIOD - 1) height = self.nodes[0].getblockcount() - assert((height % VB_PERIOD) == VB_PERIOD - 2) + assert (height % VB_PERIOD) == VB_PERIOD - 2 assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') @@ -622,7 +596,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_message["getdata"].inv[0].type == 1) + assert self.old_node.last_message["getdata"].inv[0].type == 1 # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -668,7 +642,7 @@ class SegWitTest(BitcoinTestFramework): # Mine it on test_node to create the confirmed output. test_transaction_acceptance(self.nodes[0], self.test_node, p2sh_tx, with_witness=True, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Now test standardness of v0 P2WSH outputs. # Start by creating a transaction with two outputs. @@ -699,7 +673,7 @@ class SegWitTest(BitcoinTestFramework): tx3 = CTransaction() # tx and tx2 were both accepted. Don't bother trying to reclaim the # P2PKH output; just send tx's first output back to an anyone-can-spend. - sync_mempools([self.nodes[0], self.nodes[1]]) + self.sync_mempools([self.nodes[0], self.nodes[1]]) tx3.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] tx3.vout = [CTxOut(tx.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))] tx3.wit.vtxinwit.append(CTxInWitness()) @@ -708,17 +682,17 @@ class SegWitTest(BitcoinTestFramework): if self.segwit_status != 'active': # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -756,7 +730,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True, with_witness=True) - sync_blocks(self.nodes) + self.sync_blocks() # Now test attempts to spend the output. spend_tx = CTransaction() @@ -777,7 +751,7 @@ class SegWitTest(BitcoinTestFramework): spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) spend_tx.rehash() with self.nodes[0].assert_debug_log( - expected_msgs=('Not relaying invalid transaction {}'.format(spend_tx.hash), 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')): + expected_msgs=(spend_tx.hash, 'was not accepted: mandatory-script-verify-flag-failed (Script evaluated without error but finished with a false/empty top stack element)')): test_transaction_acceptance(self.nodes[0], self.test_node, spend_tx, with_witness=False, accepted=False) # Now put the witness script in the witness, should succeed after @@ -814,7 +788,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # Test the test -- witness serialization should be different - assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) + assert msg_witness_block(block).serialize() != msg_block(block).serialize() # This empty block should be valid. test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -825,7 +799,7 @@ class SegWitTest(BitcoinTestFramework): block_2.solve() # The commitment should have changed! - assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) + assert block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1] # This should also be valid. test_witness_block(self.nodes[0], self.test_node, block_2, accepted=True) @@ -872,7 +846,7 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() block_3.rehash() - assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns + assert len(block_3.vtx[0].vout) == 4 # 3 OP_returns block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=True) @@ -903,19 +877,19 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a' * 5000000) - assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) > MAX_BLOCK_BASE_SIZE # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() != block.hash) + assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() == block.hash) + assert self.nodes[0].getbestblockhash() == block.hash # Now make sure that malleating the witness reserved value doesn't # result in a block permanently marked bad. @@ -940,7 +914,7 @@ class SegWitTest(BitcoinTestFramework): # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() - assert(len(self.utxo) > 0) + assert len(self.utxo) > 0 # Create a P2WSH transaction. # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. @@ -962,7 +936,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_OUTPUTS): parent_tx.vout.append(CTxOut(child_value, script_pubkey)) parent_tx.vout[0].nValue -= 50000 - assert(parent_tx.vout[0].nValue > 0) + assert parent_tx.vout[0].nValue > 0 parent_tx.rehash() child_tx = CTransaction() @@ -992,7 +966,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) # Make sure that our test case would exceed the old max-network-message # limit - assert(len(block.serialize(True)) > 2 * 1024 * 1024) + assert len(block.serialize(True)) > 2 * 1024 * 1024 test_witness_block(self.nodes[0], self.test_node, block, accepted=False) @@ -1002,7 +976,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) == MAX_BLOCK_BASE_SIZE test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -1020,14 +994,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert(self.nodes[0].getbestblockhash() != block.hash) + self.nodes[0].submitblock(block.serialize(True).hex()) + assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -1042,9 +1016,9 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) + self.nodes[0].submitblock(block_2.serialize(True).hex()) # Tip should not advance! - assert(self.nodes[0].getbestblockhash() != block_2.hash) + assert self.nodes[0].getbestblockhash() != block_2.hash @subtest def test_extra_witness_data(self): @@ -1164,7 +1138,7 @@ class SegWitTest(BitcoinTestFramework): # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) - assert(len(long_witness_program) == MAX_PROGRAM_LENGTH + 1) + assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1188,7 +1162,7 @@ class SegWitTest(BitcoinTestFramework): # Try again with one less byte in the witness program witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) - assert(len(witness_program) == MAX_PROGRAM_LENGTH) + assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) @@ -1219,7 +1193,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(10): tx.vout.append(CTxOut(int(value / 10), script_pubkey)) tx.vout[0].nValue -= 1000 - assert(tx.vout[0].nValue >= 0) + assert tx.vout[0].nValue >= 0 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) @@ -1369,8 +1343,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(raw_tx["vsize"], vsize) assert_equal(raw_tx["weight"], weight) assert_equal(len(raw_tx["vin"][0]["txinwitness"]), 1) - assert_equal(raw_tx["vin"][0]["txinwitness"][0], hexlify(witness_program).decode('ascii')) - assert(vsize != raw_tx["size"]) + assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_program.hex()) + assert vsize != raw_tx["size"] # Cleanup: mine the transactions and update utxo for next test self.nodes[0].generate(1) @@ -1401,7 +1375,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_SEGWIT_VERSIONS): self.utxo.append(UTXO(tx.sha256, i, split_value)) - sync_blocks(self.nodes) + self.sync_blocks() temp_utxo = [] tx = CTransaction() witness_program = CScript([OP_TRUE]) @@ -1419,8 +1393,8 @@ class SegWitTest(BitcoinTestFramework): temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) self.nodes[0].generate(1) # Mine all the transactions - sync_blocks(self.nodes) - assert(len(self.nodes[0].getrawmempool()) == 0) + self.sync_blocks() + assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 1 transactions # are non-standard @@ -1456,7 +1430,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2, tx3]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() # Add utxo to our list self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) @@ -1484,7 +1458,7 @@ class SegWitTest(BitcoinTestFramework): # Now test a premature spend. self.nodes[0].generate(98) - sync_blocks(self.nodes) + self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=False) @@ -1494,7 +1468,7 @@ class SegWitTest(BitcoinTestFramework): block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() @subtest def test_uncompressed_pubkey(self): @@ -1505,10 +1479,9 @@ class SegWitTest(BitcoinTestFramework): # Segwit transactions using uncompressed pubkeys are not accepted # under default policy, but should still pass consensus. - key = CECKey() - key.set_secretbytes(b"9") - key.set_compressed(False) - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate(False) + pubkey = key.get_pubkey().get_bytes() assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey utxo = self.utxo.pop(0) @@ -1538,7 +1511,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_wsh)) script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[0].scriptWitness.stack = [signature, pubkey] tx2.rehash() @@ -1592,7 +1565,7 @@ class SegWitTest(BitcoinTestFramework): tx5.vin.append(CTxIn(COutPoint(tx4.sha256, 0), b"")) tx5.vout.append(CTxOut(tx4.vout[0].nValue - 1000, CScript([OP_TRUE]))) (sig_hash, err) = SignatureHash(script_pubkey, tx5, 0, SIGHASH_ALL) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx5.vin[0].scriptSig = CScript([signature, pubkey]) tx5.rehash() # Should pass policy and consensus. @@ -1605,9 +1578,9 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_signature_version_1(self): - key = CECKey() - key.set_secretbytes(b"9") - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) witness_hash = sha256(witness_program) @@ -1624,7 +1597,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - sync_blocks(self.nodes) + self.sync_blocks() self.utxo.pop(0) # Test each hashtype @@ -1689,7 +1662,7 @@ class SegWitTest(BitcoinTestFramework): # Create a slight bias for producing more utxos num_outputs = random.randint(1, 11) random.shuffle(temp_utxos) - assert(len(temp_utxos) > num_inputs) + assert len(temp_utxos) > num_inputs tx = CTransaction() total_value = 0 for i in range(num_inputs): @@ -1742,7 +1715,7 @@ class SegWitTest(BitcoinTestFramework): script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL # Check that we can't have a scriptSig tx2.vin[0].scriptSig = CScript([signature, pubkey]) @@ -1803,7 +1776,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() test_transaction_acceptance(self.nodes[0], self.test_node, tx, False, True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # We'll add an unnecessary witness to this transaction that would cause # it to be non-standard, to test that violating policy with a witness @@ -1832,7 +1805,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx3, False, True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Update our utxo list; we spent the first entry. self.utxo.pop(0) @@ -1868,7 +1841,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # Creating transactions for tests p2wsh_txs = [] @@ -1932,7 +1905,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node # Valid but non-standard transactions in a block should be accepted by standard node - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -1947,10 +1920,10 @@ class SegWitTest(BitcoinTestFramework): self.start_node(2, extra_args=["-vbparams=segwit:0:999999999999"]) connect_nodes(self.nodes[0], 2) - sync_blocks(self.nodes) + self.sync_blocks() # Make sure that this peer thinks segwit has activated. - assert(get_bip9_status(self.nodes[2], 'segwit')['status'] == "active") + assert get_bip9_status(self.nodes[2], 'segwit')['status'] == "active" # Make sure this peer's blocks match those of node0. height = self.nodes[2].getblockcount() @@ -1977,7 +1950,7 @@ class SegWitTest(BitcoinTestFramework): extra_sigops_available = MAX_SIGOP_COST % sigops_per_script # We chose the number of checkmultisigs/checksigs to make this work: - assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT + assert extra_sigops_available < 100 # steer clear of MAX_OPS_PER_SCRIPT # This script, when spent with the first # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, @@ -2044,7 +2017,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block_4, accepted=True) # Reset the tip back down for the next test - sync_blocks(self.nodes) + self.sync_blocks() for x in self.nodes: x.invalidateblock(block_4.hash) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index c7ae57de86..161b67e6d0 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. @@ -103,7 +103,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - sync_blocks, wait_until, ) @@ -225,7 +224,7 @@ class SendHeadersTest(BitcoinTestFramework): # make sure all invalidated blocks are node0's self.nodes[0].generatetoaddress(length, self.nodes[0].get_deterministic_priv_key().address) - sync_blocks(self.nodes, wait=0.1) + self.sync_blocks(self.nodes, wait=0.1) for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_block_announcements() @@ -234,7 +233,7 @@ class SendHeadersTest(BitcoinTestFramework): hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) all_hashes = self.nodes[1].generatetoaddress(length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain - sync_blocks(self.nodes, wait=0.1) + self.sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): @@ -490,7 +489,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now announce a header that forks the last two blocks tip = blocks[0].sha256 - height -= 1 + height -= 2 blocks = [] # Create extra blocks for later diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 2459a9f243..02ceec3dc1 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -14,11 +14,11 @@ - Wait 1 second - Assert that we're connected - Send a ping to no_verack_node and no_version_node -- Wait 30 seconds +- Wait 1 second - Assert that we're still connected - Send a ping to no_verack_node and no_version_node -- Wait 31 seconds -- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds) +- Wait 2 seconds +- Assert that we're no longer connected (timeout to receive version/verack is 3 seconds) """ from time import sleep @@ -36,6 +36,8 @@ class TimeoutsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + # set timeout to receive version/verack to 3 seconds + self.extra_args = [["-peertimeout=3"]] def run_test(self): # Setup the p2p connections @@ -52,7 +54,7 @@ class TimeoutsTest(BitcoinTestFramework): no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) - sleep(30) + sleep(1) assert "version" in no_verack_node.last_message @@ -63,11 +65,21 @@ class TimeoutsTest(BitcoinTestFramework): no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) - sleep(31) - - assert not no_verack_node.is_connected - assert not no_version_node.is_connected - assert not no_send_node.is_connected + expected_timeout_logs = [ + "version handshake timeout from 0", + "socket no message in first 3 seconds, 1 0 from 1", + "socket no message in first 3 seconds, 0 0 from 2", + ] + + with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): + sleep(3) + # By now, we waited a total of 5 seconds. Off-by-two for two + # reasons: + # * The internal precision is one second + # * Account for network delay + assert not no_verack_node.is_connected + assert not no_version_node.is_connected + assert not no_send_node.is_connected if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 11299cbc00..534d275c28 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. @@ -57,7 +57,11 @@ from test_framework.blocktools import create_block, create_coinbase, create_tx_w from test_framework.messages import CBlockHeader, CInv, msg_block, msg_headers, msg_inv from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, sync_blocks +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, +) class AcceptBlockTest(BitcoinTestFramework): @@ -114,7 +118,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. @@ -131,7 +135,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h2f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found # But this block should be accepted by node since it has equal work. self.nodes[0].getblock(block_h2f.hash) @@ -150,7 +154,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found self.nodes[0].getblock(block_h3.hash) # But this block should be accepted by node since it has more work. @@ -263,7 +267,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_292.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) @@ -302,7 +306,7 @@ class AcceptBlockTest(BitcoinTestFramework): # 9. Connect node1 to node0 and ensure it is able to sync connect_nodes(self.nodes[0], 1) - sync_blocks([self.nodes[0], self.nodes[1]]) + self.sync_blocks([self.nodes[0], self.nodes[1]]) self.log.info("Successfully synced nodes 1 and 0") if __name__ == '__main__': diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 53916d5290..acc7cd811c 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" @@ -48,9 +48,12 @@ class RPCBindTest(BitcoinTestFramework): at a non-localhost IP. ''' self.log.info("Allow IP test for %s:%d" % (rpchost, rpcport)) - base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] + node_args = \ + ['-disablewallet', '-nolisten'] + \ + ['-rpcallowip='+x for x in allow_ips] + \ + ['-rpcbind='+addr for addr in ['127.0.0.1', "%s:%d" % (rpchost, rpcport)]] # Bind to localhost as well so start_nodes doesn't hang self.nodes[0].rpchost = None - self.start_nodes([base_args]) + self.start_nodes([node_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(self.nodes[0].datadir, 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir) node.getnetworkinfo() @@ -67,7 +70,7 @@ class RPCBindTest(BitcoinTestFramework): self.log.info("Check for ipv6") have_ipv6 = test_ipv6_local() - if not have_ipv6 and not self.options.run_ipv4: + if not have_ipv6 and not (self.options.run_ipv4 or self.options.run_nonloopback): raise SkipTest("This test requires ipv6 support.") self.log.info("Check for non-loopback interface") @@ -101,9 +104,9 @@ class RPCBindTest(BitcoinTestFramework): # check default without rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(None, '127.0.0.1', [], [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) - # check default with rpcallowip (IPv6 any) + # check default with rpcallowip (IPv4 and IPv6 localhost) self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], - [('::0', self.defaultport)]) + [('127.0.0.1', self.defaultport), ('::1', self.defaultport)]) # check only IPv6 localhost (explicit) self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], [('::1', self.defaultport)]) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 31e60f1cea..facb05b54c 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. @@ -35,6 +35,7 @@ from test_framework.util import ( from test_framework.blocktools import ( create_block, create_coinbase, + TIME_GENESIS_BLOCK, ) from test_framework.messages import ( msg_block, @@ -46,9 +47,11 @@ from test_framework.mininode import ( class BlockchainTest(BitcoinTestFramework): def set_test_params(self): + self.setup_clean_chain = True self.num_nodes = 1 def run_test(self): + self.mine_chain() self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) # Set extra args with pruning after rescan is complete self._test_getblockchaininfo() @@ -61,6 +64,15 @@ class BlockchainTest(BitcoinTestFramework): self._test_waitforblockheight() assert self.nodes[0].verifychain(4, 0) + def mine_chain(self): + self.log.info('Create some old blocks') + address = self.nodes[0].get_deterministic_priv_key().address + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + # ten-minute steps from genesis block time + self.nodes[0].setmocktime(t) + self.nodes[0].generatetoaddress(1, address) + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -160,9 +172,9 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(chaintxstats['txcount'], 2) assert_equal(chaintxstats['window_final_block_hash'], b1_hash) assert_equal(chaintxstats['window_block_count'], 0) - assert('window_tx_count' not in chaintxstats) - assert('window_interval' not in chaintxstats) - assert('txrate' not in chaintxstats) + assert 'window_tx_count' not in chaintxstats + assert 'window_interval' not in chaintxstats + assert 'txrate' not in chaintxstats def _test_gettxoutsetinfo(self): node = self.nodes[0] diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 3cc35a7b9a..7abcd71bb8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test transaction signing using the signrawtransaction* RPCs.""" +"""Test multisig RPCs""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, +) import decimal + class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -17,29 +21,40 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def get_keys(self): node0, node1, node2 = self.nodes - self.add = [node1.getnewaddress() for _ in range(self.nkeys)] - self.pub = [node1.getaddressinfo(a)["pubkey"] for a in self.add] - self.priv = [node1.dumpprivkey(a) for a in self.add] + add = [node1.getnewaddress() for _ in range(self.nkeys)] + self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add] + self.priv = [node1.dumpprivkey(a) for a in add] self.final = node2.getnewaddress() def run_test(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes - # 50 BTC each, rest will be 25 BTC each + self.check_addmultisigaddress_errors() + + self.log.info('Generating blocks ...') node0.generate(149) self.sync_all() self.moved = 0 - for self.nkeys in [3,5]: - for self.nsigs in [2,3]: + for self.nkeys in [3, 5]: + for self.nsigs in [2, 3]: for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: self.get_keys() self.do_multisig() self.checkbalances() + def check_addmultisigaddress_errors(self): + self.log.info('Check that addmultisigaddress fails when the private keys are missing') + addresses = [self.nodes[1].getnewaddress(address_type='legacy') for _ in range(2)] + assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) + for a in addresses: + # Importing all addresses should not change the result + self.nodes[0].importaddress(a) + assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) + def checkbalances(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes node0.generate(100) self.sync_all() @@ -49,13 +64,13 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 - total = 149*50 + (height-149-100)*25 + total = 149 * 50 + (height - 149 - 100) * 25 assert bal1 == 0 assert bal2 == self.moved - assert bal0+bal1+bal2 == total + assert bal0 + bal1 + bal2 == total def do_multisig(self): - node0,node1,node2 = self.nodes + node0, node1, node2 = self.nodes msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) madd = msig["address"] @@ -74,7 +89,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txid = node0.sendtoaddress(madd, 40) tx = node0.getrawtransaction(txid, True) - vout = [v["n"] for v in tx["vout"] if madd in v["scriptPubKey"].get("addresses",[])] + vout = [v["n"] for v in tx["vout"] if madd in v["scriptPubKey"].get("addresses", [])] assert len(vout) == 1 vout = vout[0] scriptPubKey = tx["vout"][vout]["scriptPubKey"]["hex"] @@ -86,16 +101,17 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): outval = value - decimal.Decimal("0.00001000") rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) - rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], prevtxs) + rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs - 1], prevtxs) rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) self.moved += outval - tx = node0.sendrawtransaction(rawtx3["hex"], True) + tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = node0.generate(1)[0] assert tx in node0.getblock(blk)["tx"] txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 940386eee7..01b8cb1854 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" from test_framework.messages import CTransaction, sha256 from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, hex_str_to_bytes from io import BytesIO @@ -81,7 +81,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(multisig_script) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH - multisig_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(multisig_script))) + multisig_script_hash = sha256(hex_str_to_bytes(multisig_script)).hex() assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) # 4) P2SH scriptPubKey @@ -119,7 +119,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(cltv_script) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH - cltv_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(cltv_script))) + cltv_script_hash = sha256(hex_str_to_bytes(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) # 7) P2PK scriptPubKey @@ -196,7 +196,7 @@ class DecodeScriptTest(BitcoinTestFramework): # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). - push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] + push_signature = txSave.vin[0].scriptSig.hex()[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' @@ -206,23 +206,23 @@ class DecodeScriptTest(BitcoinTestFramework): # 1) P2PK scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) def run_test(self): diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 588bfbe083..9a21998d11 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,32 +1,29 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error +# from test_framework.util import assert_raises_rpc_error class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], ["-deprecatedrpc=generate"]] - - def skip_test_if_missing_module(self): - # The generate RPC method requires the wallet to be compiled - self.skip_if_no_wallet() + self.extra_args = [[], []] def run_test(self): # This test should be used to verify correct behaviour of deprecated # RPC methods with and without the -deprecatedrpc flags. For example: # - # self.log.info("Make sure that -deprecatedrpc=createmultisig allows it to take addresses") - # assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, [self.nodes[0].getnewaddress()]) - # self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) - - self.log.info("Test generate RPC") - assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) - self.nodes[1].generate(1) + # In set_test_params: + # self.extra_args = [[], ["-deprecatedrpc=generate"]] + # + # In run_test: + # self.log.info("Test generate RPC") + # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) + # self.nodes[1].generate(1) + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py new file mode 100755 index 0000000000..1984694692 --- /dev/null +++ b/test/functional/rpc_deriveaddresses.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the deriveaddresses rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create +from test_framework.util import assert_equal, assert_raises_rpc_error + +class DeriveaddressesTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.supports_cli = 1 + + def run_test(self): + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a") + + descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)#t6wfjs64" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + assert_equal(self.nodes[0].deriveaddresses(descriptor), [address]) + + descriptor = descriptor[:-9] + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, descriptor) + + descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)#s9ga3alw" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) + + ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)#kft60nuy" + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, [1, 2]), ["bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), [0, 2]) + + assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)")) + + assert_raises_rpc_error(-8, "End of range is too high", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 10000000000) + + assert_raises_rpc_error(-8, "Range is too large", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [1000000000, 2000000000]) + + assert_raises_rpc_error(-8, "Range specified as [begin,end] must not have begin after end", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [2, 0]) + + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [-1, 0]) + + combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)") + assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) + + hardened_without_privkey_descriptor = descsum_create("wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)") + assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) + + bare_multisig_descriptor = descsum_create("multi(1,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)") + assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor) + +if __name__ == '__main__': + DeriveaddressesTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 0c61e9ab62..d89fd6461f 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -32,7 +32,7 @@ class RawTransactionsTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.setup_nodes() connect_nodes_bi(self.nodes, 0, 1) @@ -94,7 +94,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test that we have enough inputs + assert len(dec_tx['vin']) > 0 #test that we have enough inputs ############################## # simple test with two coins # @@ -107,7 +107,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test if we have enough inputs + assert len(dec_tx['vin']) > 0 #test if we have enough inputs ############################## # simple test with two coins # @@ -120,7 +120,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -139,7 +139,7 @@ class RawTransactionsTest(BitcoinTestFramework): for out in dec_tx['vout']: totalOut += out['value'] - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -363,7 +363,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ ############################################################ @@ -378,7 +378,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -405,7 +405,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -438,7 +438,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -558,7 +558,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance*19) #~19 inputs + assert feeDelta >= 0 and feeDelta <= feeTolerance*19 #~19 inputs ############################################# @@ -620,7 +620,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(res_dec["vin"]), 1) assert_equal(res_dec["vin"][0]["txid"], watchonly_txid) - assert("fee" in result.keys()) + assert "fee" in result.keys() assert_greater_than(result["changepos"], -1) ############################################################### @@ -635,16 +635,16 @@ class RawTransactionsTest(BitcoinTestFramework): result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) - assert(res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid) + assert res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid assert_greater_than(result["fee"], 0) assert_greater_than(result["changepos"], -1) assert_equal(result["fee"] + res_dec["vout"][result["changepos"]]["value"], watchonly_amount / 10) signedtx = self.nodes[3].signrawtransactionwithwallet(result["hex"]) - assert(not signedtx["complete"]) + assert not signedtx["complete"] signedtx = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) - assert(signedtx["complete"]) + assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) self.nodes[0].generate(1) self.sync_all() @@ -676,10 +676,10 @@ class RawTransactionsTest(BitcoinTestFramework): for out in res_dec['vout']: if out['value'] > 1.0: changeaddress += out['scriptPubKey']['addresses'][0] - assert(changeaddress != "") + assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() # Now the change address key should be removed from the keypool - assert(changeaddress != nextaddr) + assert changeaddress != nextaddr ###################################### # Test subtractFeeFromOutputs option # diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py new file mode 100755 index 0000000000..bd93b6f7a4 --- /dev/null +++ b/test/functional/rpc_getblockfilter.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getblockfilter RPC.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, assert_is_hex_string, assert_raises_rpc_error, + connect_nodes, disconnect_nodes, sync_blocks + ) + +FILTER_TYPES = ["basic"] + +class GetBlockFilterTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [["-blockfilterindex"], []] + + def run_test(self): + # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting + disconnect_nodes(self.nodes[0], 1) + + self.nodes[0].generate(3) + self.nodes[1].generate(4) + + assert_equal(self.nodes[0].getblockcount(), 3) + chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Reorg node 0 to a new chain + connect_nodes(self.nodes[0], 1) + sync_blocks(self.nodes) + + assert_equal(self.nodes[0].getblockcount(), 4) + chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Test getblockfilter returns a filter for all blocks and filter types on active chain + for block_hash in chain1_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter returns a filter for all blocks and filter types on stale chain + for block_hash in chain0_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter with unknown block + bad_block_hash = "0123456789abcdef" * 4 + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockfilter, bad_block_hash, "basic") + + # Test getblockfilter with undefined filter type + genesis_hash = self.nodes[0].getblockhash(0) + assert_raises_rpc_error(-5, "Unknown filtertype", self.nodes[0].getblockfilter, genesis_hash, "unknown") + +if __name__ == '__main__': + GetBlockFilterTest().main() diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index ca9e24367a..e17a8f6421 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -178,5 +178,10 @@ class GetblockstatsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats, hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f') + # Invalid number of args + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats, '00', 1, 2) + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats) + + if __name__ == '__main__': GetblockstatsTest().main() diff --git a/test/functional/rpc_getchaintips.py b/test/functional/rpc_getchaintips.py index c869c7262f..8dc8474374 100755 --- a/test/functional/rpc_getchaintips.py +++ b/test/functional/rpc_getchaintips.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getchaintips RPC. @@ -28,7 +28,8 @@ class GetChainTipsTest (BitcoinTestFramework): self.split_network() self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) self.nodes[2].generatetoaddress(20, self.nodes[2].get_deterministic_priv_key().address) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) tips = self.nodes[1].getchaintips () assert_equal (len (tips), 1) diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index d8ecdd573a..3d3f694fd3 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" -import time - from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, connect_nodes_bi, sync_blocks +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.util import ( + assert_equal, + connect_nodes_bi, + wait_until, +) + class InvalidateTest(BitcoinTestFramework): def set_test_params(self): @@ -21,46 +25,66 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") self.log.info("Mine 4 blocks on Node 0") self.nodes[0].generatetoaddress(4, self.nodes[0].get_deterministic_priv_key().address) - assert(self.nodes[0].getblockcount() == 4) - besthash = self.nodes[0].getbestblockhash() + assert_equal(self.nodes[0].getblockcount(), 4) + besthash_n0 = self.nodes[0].getbestblockhash() self.log.info("Mine competing 6 blocks on Node 1") self.nodes[1].generatetoaddress(6, self.nodes[1].get_deterministic_priv_key().address) - assert(self.nodes[1].getblockcount() == 6) + assert_equal(self.nodes[1].getblockcount(), 6) self.log.info("Connect nodes to force a reorg") - connect_nodes_bi(self.nodes,0,1) - sync_blocks(self.nodes[0:2]) - assert(self.nodes[0].getblockcount() == 6) + connect_nodes_bi(self.nodes, 0, 1) + self.sync_blocks(self.nodes[0:2]) + assert_equal(self.nodes[0].getblockcount(), 6) badhash = self.nodes[1].getblockhash(2) self.log.info("Invalidate block 2 on node 0 and verify we reorg to node 0's original chain") self.nodes[0].invalidateblock(badhash) - newheight = self.nodes[0].getblockcount() - newhash = self.nodes[0].getbestblockhash() - if (newheight != 4 or newhash != besthash): - raise AssertionError("Wrong tip for node0, hash %s, height %d"%(newhash,newheight)) + assert_equal(self.nodes[0].getblockcount(), 4) + assert_equal(self.nodes[0].getbestblockhash(), besthash_n0) self.log.info("Make sure we won't reorg to a lower work chain:") - connect_nodes_bi(self.nodes,1,2) + connect_nodes_bi(self.nodes, 1, 2) self.log.info("Sync node 2 to node 1 so both have 6 blocks") - sync_blocks(self.nodes[1:3]) - assert(self.nodes[2].getblockcount() == 6) + self.sync_blocks(self.nodes[1:3]) + assert_equal(self.nodes[2].getblockcount(), 6) self.log.info("Invalidate block 5 on node 1 so its tip is now at 4") self.nodes[1].invalidateblock(self.nodes[1].getblockhash(5)) - assert(self.nodes[1].getblockcount() == 4) + assert_equal(self.nodes[1].getblockcount(), 4) self.log.info("Invalidate block 3 on node 2, so its tip is now 2") self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3)) - assert(self.nodes[2].getblockcount() == 2) + assert_equal(self.nodes[2].getblockcount(), 2) self.log.info("..and then mine a block") self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") - time.sleep(5) - assert_equal(self.nodes[2].getblockcount(), 3) - assert_equal(self.nodes[0].getblockcount(), 4) - node1height = self.nodes[1].getblockcount() - if node1height < 4: - raise AssertionError("Node 1 reorged to a lower height: %d"%node1height) + wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) + wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) + wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) + + self.log.info("Verify that we reconsider all ancestors as well") + blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + # Invalidate the two blocks at the tip + self.nodes[1].invalidateblock(blocks[-1]) + self.nodes[1].invalidateblock(blocks[-2]) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-3]) + # Reconsider only the previous tip + self.nodes[1].reconsiderblock(blocks[-1]) + # Should be back at the tip by now + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + + self.log.info("Verify that we reconsider all descendants") + blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + # Invalidate the two blocks at the tip + self.nodes[1].invalidateblock(blocks[-2]) + self.nodes[1].invalidateblock(blocks[-4]) + assert_equal(self.nodes[1].getbestblockhash(), blocks[-5]) + # Reconsider only the previous tip + self.nodes[1].reconsiderblock(blocks[-4]) + # Should be back at the tip by now + assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) + if __name__ == '__main__': InvalidateTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py new file mode 100755 index 0000000000..7bf8e68176 --- /dev/null +++ b/test/functional/rpc_misc.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPC misc output.""" +import xml.etree.ElementTree as ET + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, +) + +from test_framework.authproxy import JSONRPCException + + +class RpcMiscTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + node = self.nodes[0] + + self.log.info("test getmemoryinfo") + memory = node.getmemoryinfo()['locked'] + assert_greater_than(memory['used'], 0) + assert_greater_than(memory['free'], 0) + assert_greater_than(memory['total'], 0) + # assert_greater_than_or_equal() for locked in case locking pages failed at some point + assert_greater_than_or_equal(memory['locked'], 0) + assert_greater_than(memory['chunks_used'], 0) + assert_greater_than(memory['chunks_free'], 0) + assert_equal(memory['used'] + memory['free'], memory['total']) + + self.log.info("test mallocinfo") + try: + mallocinfo = node.getmemoryinfo(mode="mallocinfo") + self.log.info('getmemoryinfo(mode="mallocinfo") call succeeded') + tree = ET.fromstring(mallocinfo) + assert_equal(tree.tag, 'malloc') + except JSONRPCException: + self.log.info('getmemoryinfo(mode="mallocinfo") not available') + assert_raises_rpc_error(-8, 'mallocinfo is only available when compiled with glibc 2.10+', node.getmemoryinfo, mode="mallocinfo") + + assert_raises_rpc_error(-8, "unknown mode foobar", node.getmemoryinfo, mode="foobar") + +if __name__ == '__main__': + RpcMiscTest().main() diff --git a/test/functional/rpc_named_arguments.py b/test/functional/rpc_named_arguments.py index 6c5c29dfdf..ecac9c2f82 100755 --- a/test/functional/rpc_named_arguments.py +++ b/test/functional/rpc_named_arguments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test using named arguments for RPCs.""" @@ -17,7 +17,7 @@ class NamedArgumentTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] h = node.help(command='getblockchaininfo') - assert(h.startswith('getblockchaininfo\n')) + assert h.startswith('getblockchaininfo\n') assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getblockchaininfo') diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 0affddcf05..b12eb1d9ec 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -67,8 +67,8 @@ class NetTest(BitcoinTestFramework): peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): - assert_greater_than_or_equal(after['bytesrecv_per_msg']['pong'], before['bytesrecv_per_msg']['pong'] + 32) - assert_greater_than_or_equal(after['bytessent_per_msg']['ping'], before['bytessent_per_msg']['ping'] + 32) + assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32) + assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32) def _test_getnetworkinginfo(self): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 72e6e6329f..2d5631bb27 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" @@ -8,7 +8,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, - sync_blocks, ) def unidirectional_node_sync_via_rpc(node_src, node_dest): @@ -16,7 +15,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blockhash = node_src.getbestblockhash() while True: try: - assert(len(node_dest.getblock(blockhash, False)) > 0) + assert len(node_dest.getblock(blockhash, False)) > 0 break except: blocks_to_copy.append(blockhash) @@ -24,7 +23,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blocks_to_copy.reverse() for blockhash in blocks_to_copy: blockdata = node_src.getblock(blockhash, False) - assert(node_dest.submitblock(blockdata) in (None, 'inconclusive')) + assert node_dest.submitblock(blockdata) in (None, 'inconclusive') def node_sync_via_rpc(nodes): for node_src in nodes: @@ -57,7 +56,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine competing blocks E-F-G on Node 1") hashG = self.nodes[1].generatetoaddress(3, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 5) - assert(hashC != hashG) + assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) @@ -72,7 +71,7 @@ class PreciousTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbestblockhash(), hashC) self.log.info("Make Node1 prefer block C") self.nodes[1].preciousblock(hashC) - sync_blocks(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC + self.sync_blocks(self.nodes[0:2]) # wait because node 1 may not have downloaded hashC assert_equal(self.nodes[1].getbestblockhash(), hashC) self.log.info("Make Node1 prefer block G again") self.nodes[1].preciousblock(hashG) @@ -86,7 +85,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") self.nodes[0].generatetoaddress(1, gen_address(0)) assert_equal(self.nodes[0].getblockcount(), 6) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() assert_equal(self.nodes[1].getbestblockhash(), hashH) self.log.info("Node1 should not be able to prefer block C anymore") diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 6f9f52f001..8a7ea7aa58 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. """ +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, find_output, disconnect_nodes, connect_nodes_bi, sync_blocks +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes_bi, + disconnect_nodes, + find_output, +) import json import os @@ -40,22 +47,22 @@ class PSBTTest(BitcoinTestFramework): online_node.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) mining_node.generate(nblocks=1) - sync_blocks([mining_node, online_node]) + self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) utxos = online_node.listunspent(addresses=[offline_addr]) raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] - assert("non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]) + assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] - assert("witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0]) + assert "witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) mining_node.generate(1) - sync_blocks([mining_node, online_node]) + self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) # Reconnect @@ -159,11 +166,11 @@ class PSBTTest(BitcoinTestFramework): node1_addr = self.nodes[1].getnewaddress() node2_addr = self.nodes[2].getnewaddress() txid1 = self.nodes[0].sendtoaddress(node1_addr, 13) - txid2 =self.nodes[0].sendtoaddress(node2_addr, 13) - self.nodes[0].generate(6) + txid2 = self.nodes[0].sendtoaddress(node2_addr, 13) + blockhash = self.nodes[0].generate(6)[0] self.sync_all() - vout1 = find_output(self.nodes[1], txid1, 13) - vout2 = find_output(self.nodes[2], txid2, 13) + vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash) + vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash) # Create a psbt spending outputs from nodes 1 and 2 psbt_orig = self.nodes[0].createpsbt([{"txid":txid1, "vout":vout1}, {"txid":txid2, "vout":vout2}], {self.nodes[0].getnewaddress():25.999}) @@ -192,8 +199,8 @@ class PSBTTest(BitcoinTestFramework): psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"replaceable":True}, False) decoded_psbt = self.nodes[0].decodepsbt(psbtx_info["psbt"]) for tx_in, psbt_in in zip(decoded_psbt["tx"]["vin"], decoded_psbt["inputs"]): - assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) - assert "bip32_derivs" not in psbt_in + assert_equal(tx_in["sequence"], MAX_BIP125_RBF_SEQUENCE) + assert "bip32_derivs" not in psbt_in assert_equal(decoded_psbt["tx"]["locktime"], block_height+2) # Same construction with only locktime set @@ -211,6 +218,10 @@ class PSBTTest(BitcoinTestFramework): assert tx_in["sequence"] > MAX_BIP125_RBF_SEQUENCE assert_equal(decoded_psbt["tx"]["locktime"], 0) + # Make sure change address wallet does not have P2SH innerscript access to results in success + # when attempting BnB coin selection + self.nodes[0].walletcreatefundedpsbt([], [{self.nodes[2].getnewaddress():unspent["amount"]+1}], block_height+2, {"changeAddress":self.nodes[1].getnewaddress()}, False) + # Regression test for 14473 (mishandling of already-signed witness transaction): psbtx_info = self.nodes[0].walletcreatefundedpsbt([{"txid":unspent["txid"], "vout":unspent["vout"]}], [{self.nodes[2].getnewaddress():unspent["amount"]+1}]) complete_psbt = self.nodes[0].walletprocesspsbt(psbtx_info["psbt"]) @@ -264,6 +275,9 @@ class PSBTTest(BitcoinTestFramework): combined = self.nodes[2].combinepsbt(combiner['combine']) assert_equal(combined, combiner['result']) + # Empty combiner test + assert_raises_rpc_error(-8, "Parameter 'txs' cannot be empty", self.nodes[0].combinepsbt, []) + # Finalizer test for finalizer in finalizers: finalized = self.nodes[2].finalizepsbt(finalizer['finalize'], False)['psbt'] @@ -285,5 +299,75 @@ class PSBTTest(BitcoinTestFramework): psbt = self.nodes[1].walletcreatefundedpsbt([], [{p2pkh : 1}], 0, {"includeWatching" : True}, True) self.nodes[0].decodepsbt(psbt['psbt']) + # Test decoding error: invalid base64 + assert_raises_rpc_error(-22, "TX decode failed invalid base64", self.nodes[0].decodepsbt, ";definitely not base64;") + + # Send to all types of addresses + addr1 = self.nodes[1].getnewaddress("", "bech32") + txid1 = self.nodes[0].sendtoaddress(addr1, 11) + vout1 = find_output(self.nodes[0], txid1, 11) + addr2 = self.nodes[1].getnewaddress("", "legacy") + txid2 = self.nodes[0].sendtoaddress(addr2, 11) + vout2 = find_output(self.nodes[0], txid2, 11) + addr3 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid3 = self.nodes[0].sendtoaddress(addr3, 11) + vout3 = find_output(self.nodes[0], txid3, 11) + self.sync_all() + + # Update a PSBT with UTXOs from the node + # Bech32 inputs should be filled with witness UTXO. Other inputs should not be filled because they are non-witness + psbt = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1},{"txid":txid2, "vout":vout2},{"txid":txid3, "vout":vout3}], {self.nodes[0].getnewaddress():32.999}) + decoded = self.nodes[1].decodepsbt(psbt) + assert "witness_utxo" not in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + updated = self.nodes[1].utxoupdatepsbt(psbt) + decoded = self.nodes[1].decodepsbt(updated) + assert "witness_utxo" in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + + # Two PSBTs with a common input should not be joinable + psbt1 = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1}], {self.nodes[0].getnewaddress():Decimal('10.999')}) + assert_raises_rpc_error(-8, "exists in multiple PSBTs", self.nodes[1].joinpsbts, [psbt1, updated]) + + # Join two distinct PSBTs + addr4 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid4 = self.nodes[0].sendtoaddress(addr4, 5) + vout4 = find_output(self.nodes[0], txid4, 5) + self.nodes[0].generate(6) + self.sync_all() + psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')}) + psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt'] + psbt2_decoded = self.nodes[0].decodepsbt(psbt2) + assert "final_scriptwitness" in psbt2_decoded['inputs'][0] and "final_scriptSig" in psbt2_decoded['inputs'][0] + joined = self.nodes[0].joinpsbts([psbt, psbt2]) + joined_decoded = self.nodes[0].decodepsbt(joined) + assert len(joined_decoded['inputs']) == 4 and len(joined_decoded['outputs']) == 2 and "final_scriptwitness" not in joined_decoded['inputs'][3] and "final_scriptSig" not in joined_decoded['inputs'][3] + + # Newly created PSBT needs UTXOs and updating + addr = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid = self.nodes[0].sendtoaddress(addr, 7) + addrinfo = self.nodes[1].getaddressinfo(addr) + blockhash = self.nodes[0].generate(6)[0] + self.sync_all() + vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash) + psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')}) + analyzed = self.nodes[0].analyzepsbt(psbt) + assert not analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'updater' and analyzed['next'] == 'updater' + + # After update with wallet, only needs signing + updated = self.nodes[1].walletprocesspsbt(psbt, False, 'ALL', True)['psbt'] + analyzed = self.nodes[0].analyzepsbt(updated) + assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program'] + + # Check fee and size things + assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == Decimal('0.00746268') + + # After signing and finalizing, needs extracting + signed = self.nodes[1].walletprocesspsbt(updated)['psbt'] + analyzed = self.nodes[0].analyzepsbt(signed) + assert analyzed['inputs'][0]['has_utxo'] and analyzed['inputs'][0]['is_final'] and analyzed['next'] == 'extractor' + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 8ed490f552..6e71817dc3 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -17,7 +17,7 @@ from decimal import Decimal from io import BytesIO from test_framework.messages import CTransaction, ToHex from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes class multidict(dict): """Dictionary that allows duplicate keys. @@ -42,12 +42,12 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-addresstype=legacy"], ["-addresstype=legacy"], ["-addresstype=legacy"]] + self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): super().setup_network() connect_nodes_bi(self.nodes, 0, 2) @@ -100,6 +100,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].createrawtransaction, [], {address: -1}) assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], multidict([(address, 1), (address, 1)])) assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], [{address: 1}, {address: 1}]) + assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], [{"data": 'aa'}, {"data": "bb"}]) + assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", self.nodes[0].createrawtransaction, [], multidict([("data", 'aa'), ("data", "bb")])) assert_raises_rpc_error(-8, "Invalid parameter, key-value pair must contain exactly one key", self.nodes[0].createrawtransaction, [], [{'a': 1, 'b': 2}]) assert_raises_rpc_error(-8, "Invalid parameter, key-value pair not an object as expected", self.nodes[0].createrawtransaction, [], [['key-value pair1'], ['2']]) @@ -117,29 +119,22 @@ class RawTransactionsTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs={address: 99})))) assert_equal(len(tx.vout), 1) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))))) assert_equal(len(tx.vout), 2) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) - # Two data outputs - tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([('data', '99'), ('data', '99')]))))) - assert_equal(len(tx.vout), 2) - assert_equal( - bytes_to_hex_str(tx.serialize()), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{'data': '99'}, {'data': '99'}]), - ) # Multiple mixed outputs - tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), ('data', '99'), ('data', '99')]))))) + tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))))) assert_equal(len(tx.vout), 3) assert_equal( - bytes_to_hex_str(tx.serialize()), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {'data': '99'}, {'data': '99'}]), + tx.serialize().hex(), + self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) for type in ["bech32", "p2sh-segwit", "legacy"]: @@ -290,11 +285,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "amount" : vout['value']}] @@ -335,11 +326,7 @@ class RawTransactionsTest(BitcoinTestFramework): txDetails = self.nodes[0].gettransaction(txId, True) rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = False - for outpoint in rawTx2['vout']: - if outpoint['value'] == Decimal('2.20000000'): - vout = outpoint - break + vout = next(o for o in rawTx2['vout'] if o['value'] == Decimal('2.20000000')) bal = self.nodes[0].getbalance() inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex'], "amount" : vout['value']}] @@ -439,5 +426,30 @@ class RawTransactionsTest(BitcoinTestFramework): decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + self.log.info('sendrawtransaction/testmempoolaccept with maxfeerate') + + txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) + rawTx = self.nodes[0].getrawtransaction(txId, True) + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) + + self.sync_all() + inputs = [{ "txid" : txId, "vout" : vout['n'] }] + outputs = { self.nodes[0].getnewaddress() : Decimal("0.99999000") } # 1000 sat fee + rawTx = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) + assert_equal(rawTxSigned['complete'], True) + # 1000 sat fee, ~200 b transaction, fee rate should land around 5 sat/b = 0.00005000 BTC/kB + # Thus, testmempoolaccept should reject + testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] + assert_equal(testres['allowed'], False) + assert_equal(testres['reject-reason'], '256: absurdly-high-fee') + # and sendrawtransaction should throw + assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + # And below calls should both succeed + testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.00007000')[0] + assert_equal(testres['allowed'], True) + self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.00007000') + + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 881b839a4e..6346477922 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" @@ -10,6 +10,9 @@ from decimal import Decimal import shutil import os +def descriptors(out): + return sorted(u['desc'] for u in out['unspents']) + class ScantxoutsetTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -92,6 +95,12 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1499}])['total_amount'], Decimal("12.288")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) + assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": [1500,1500]}])['total_amount'], Decimal("16.384")) + + # Test the reported descriptors for a few matches + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa']) if __name__ == '__main__': ScantxoutsetTest().main() diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessage.py index ad0e29b451..0cb3ce4215 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing and verifying messages.""" @@ -25,18 +25,18 @@ class SignMessagesTest(BitcoinTestFramework): expected_signature = 'INbVnW4e6PeRmsv2Qgu8NuopvrVjkcxob+sX8OcZG0SALhWybUjzMLPdAsXI46YZGb0KQTRii+wWIQzRpG/U+S0=' signature = self.nodes[0].signmessagewithprivkey(priv_key, message) assert_equal(expected_signature, signature) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test signing with an address with wallet') address = self.nodes[0].getnewaddress() signature = self.nodes[0].signmessage(address, message) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test verifying with another address should not work') other_address = self.nodes[0].getnewaddress() other_signature = self.nodes[0].signmessage(other_address, message) - assert(not self.nodes[0].verifymessage(other_address, signature, message)) - assert(not self.nodes[0].verifymessage(address, other_signature, message)) + assert not self.nodes[0].verifymessage(other_address, signature, message) + assert not self.nodes[0].verifymessage(address, other_signature, message) if __name__ == '__main__': SignMessagesTest().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 291538df64..780758e219 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes +from test_framework.messages import sha256 +from test_framework.script import CScript, OP_0 +from decimal import Decimal class SignRawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [["-deprecatedrpc=signrawtransaction"]] + self.num_nodes = 2 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -143,9 +145,33 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) assert not rawTxSigned['errors'][0]['witness'] + def witness_script_test(self): + # Now test signing transaction to P2SH-P2WSH addresses without wallet + # Create a new P2SH-P2WSH 1-of-1 multisig address: + embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) + embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"]) + p2sh_p2wsh_address = self.nodes[1].addmultisigaddress(1, [embedded_address["pubkey"]], "", "p2sh-segwit") + # send transaction to P2SH-P2WSH 1-of-1 multisig address + self.nodes[0].generate(101) + self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) + self.nodes[0].generate(1) + self.sync_all() + # Find the UTXO for the transaction node[1] should have received, check witnessScript matches + unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0] + assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"]) + p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))]) + assert_equal(unspent_output["redeemScript"], p2sh_redeemScript.hex()) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() + self.witness_script_test() self.test_with_lock_outputs() diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 20b6341550..e86f91b1d0 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC call related to the uptime command. @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) - assert(self.nodes[0].uptime() >= wait_time) + assert self.nodes[0].uptime() >= wait_time if __name__ == '__main__': diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 456d43aa2e..f36cffe957 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" from .script import hash256, hash160, sha256, CScript, OP_0 -from .util import bytes_to_hex_str, hex_str_to_bytes +from .util import hex_str_to_bytes from . import segwit_addr @@ -16,9 +16,9 @@ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def byte_to_base58(b, version): result = '' - str = bytes_to_hex_str(b) - str = bytes_to_hex_str(chr(version).encode('latin-1')) + str - checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str))) + str = b.hex() + str = chr(version).encode('latin-1').hex() + str + checksum = hash256(hex_str_to_bytes(str)).hex() str += checksum[:8] value = int('0x'+str,0) while value > 0: @@ -32,12 +32,12 @@ def byte_to_base58(b, version): # TODO: def base58_decode def keyhash_to_p2pkh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 0 if main else 111 return byte_to_base58(hash, version) def scripthash_to_p2sh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 5 if main else 196 return byte_to_base58(hash, version) @@ -80,11 +80,11 @@ def check_key(key): key = hex_str_to_bytes(key) # Assuming this is hex string if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): return key - assert(False) + assert False def check_script(script): if (type(script) is str): script = hex_str_to_bytes(script) # Assuming this is hex string if (type(script) is bytes or type(script) is CScript): return script - assert(False) + assert False diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 1140fe9b3e..4ba6ac1db2 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -35,6 +35,7 @@ ServiceProxy class: import base64 import decimal +from http import HTTPStatus import http.client import json import logging @@ -49,13 +50,14 @@ USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): - def __init__(self, rpc_error): + def __init__(self, rpc_error, http_status=None): try: errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' super().__init__(errmsg) self.error = rpc_error + self.http_status = http_status def EncodeDecimal(o): @@ -120,8 +122,11 @@ class AuthServiceProxy(): def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s" % (AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("-{}-> {} {}".format( + AuthServiceProxy.__id_count, + self._service_name, + json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii), + )) if args and argsn: raise ValueError('Cannot handle both named and positional arguments') return {'version': '1.1', @@ -131,19 +136,26 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - response = self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: - raise JSONRPCException(response['error']) + raise JSONRPCException(response['error'], status) elif 'result' not in response: raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}) + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) else: return response['result'] def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) - return self._request('POST', self.__url.path, postdata.encode('utf-8')) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + return response def _get_response(self): req_start_time = time.time() @@ -162,8 +174,9 @@ class AuthServiceProxy(): content_type = http_response.getheader('Content-Type') if content_type != 'application/json': - raise JSONRPCException({ - 'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}) + raise JSONRPCException( + {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, + http_response.status) responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) @@ -172,7 +185,7 @@ class AuthServiceProxy(): log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) - return response + return response, http_response.status def __truediv__(self, relative_uri): return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 81cce1167b..7ac044d0d0 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -20,7 +20,6 @@ from .messages import ( CTxOut, FromHex, ToHex, - bytes_to_hex_str, hash256, hex_str_to_bytes, ser_string, @@ -41,12 +40,19 @@ from .script import ( from .util import assert_equal from io import BytesIO +MAX_BLOCK_SIGOPS = 20000 + +# Genesis block time (regtest) +TIME_GENESIS_BLOCK = 1296688602 + # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" -def create_block(hashprev, coinbase, ntime=None): + +def create_block(hashprev, coinbase, ntime=None, *, version=1): """Create a block (with regtest difficulty).""" block = CBlock() + block.nVersion = version if ntime is None: import time block.nTime = int(time.time() + 600) @@ -125,7 +131,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C Can optionally pass scriptPubKey and scriptSig, default is anyone-can-spend output. """ tx = CTransaction() - assert(n < len(prevtx.vout)) + assert n < len(prevtx.vout) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff)) tx.vout.append(CTxOut(amount, script_pub_key)) tx.calc_sha256() @@ -183,7 +189,7 @@ def witness_script(use_p2wsh, pubkey): witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) scripthash = sha256(witness_program) pkscript = CScript([OP_0, scripthash]) - return bytes_to_hex_str(pkscript) + return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): """Return a transaction (in hex) that spends the given utxo to a segwit output. @@ -208,7 +214,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) if (sign): signed = node.signrawtransactionwithwallet(tx_to_witness) - assert("errors" not in signed or len(["errors"]) == 0) + assert "errors" not in signed or len(["errors"]) == 0 return node.sendrawtransaction(signed["hex"]) else: if (insert_redeem_script): diff --git a/test/functional/test_framework/descriptors.py b/test/functional/test_framework/descriptors.py new file mode 100644 index 0000000000..29482ce01e --- /dev/null +++ b/test/functional/test_framework/descriptors.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Utility functions related to output descriptors""" + +INPUT_CHARSET = "0123456789()[],'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#\"\\ " +CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" +GENERATOR = [0xf5dee51989, 0xa9fdca3312, 0x1bab10e32d, 0x3706b1677a, 0x644d626ffd] + +def descsum_polymod(symbols): + """Internal function that computes the descriptor checksum.""" + chk = 1 + for value in symbols: + top = chk >> 35 + chk = (chk & 0x7ffffffff) << 5 ^ value + for i in range(5): + chk ^= GENERATOR[i] if ((top >> i) & 1) else 0 + return chk + +def descsum_expand(s): + """Internal function that does the character to symbol expansion""" + groups = [] + symbols = [] + for c in s: + if not c in INPUT_CHARSET: + return None + v = INPUT_CHARSET.find(c) + symbols.append(v & 31) + groups.append(v >> 5) + if len(groups) == 3: + symbols.append(groups[0] * 9 + groups[1] * 3 + groups[2]) + groups = [] + if len(groups) == 1: + symbols.append(groups[0]) + elif len(groups) == 2: + symbols.append(groups[0] * 3 + groups[1]) + return symbols + +def descsum_create(s): + """Add a checksum to a descriptor without""" + symbols = descsum_expand(s) + [0, 0, 0, 0, 0, 0, 0, 0] + checksum = descsum_polymod(symbols) ^ 1 + return s + '#' + ''.join(CHECKSUM_CHARSET[(checksum >> (5 * (7 - i))) & 31] for i in range(8)) + +def descsum_check(s, require=True): + """Verify that the checksum is correct in a descriptor""" + if not '#' in s: + return not require + if s[-9] != '#': + return False + if not all(x in CHECKSUM_CHARSET for x in s[-8:]): + return False + symbols = descsum_expand(s[:-9]) + [CHECKSUM_CHARSET.find(x) for x in s[-8:]] + return descsum_polymod(symbols) == 1 diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 1b3e510dc4..912c0ca978 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -1,226 +1,386 @@ -# Copyright (c) 2011 Sam Rushing -"""ECC secp256k1 OpenSSL wrapper. +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test-only secp256k1 elliptic curve implementation -WARNING: This module does not mlock() secrets; your private keys may end up on -disk in swap! Use with caution! +WARNING: This code is slow, uses bad randomness, does not properly protect +keys, and is trivially vulnerable to side channel attacks. Do not use for +anything but tests.""" +import random -This file is modified from python-bitcoinlib. -""" - -import ctypes -import ctypes.util -import hashlib - -ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library ('ssl') or 'libeay32') - -ssl.BN_new.restype = ctypes.c_void_p -ssl.BN_new.argtypes = [] - -ssl.BN_bin2bn.restype = ctypes.c_void_p -ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] - -ssl.BN_CTX_free.restype = None -ssl.BN_CTX_free.argtypes = [ctypes.c_void_p] - -ssl.BN_CTX_new.restype = ctypes.c_void_p -ssl.BN_CTX_new.argtypes = [] - -ssl.ECDH_compute_key.restype = ctypes.c_int -ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_sign.restype = ctypes.c_int -ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_verify.restype = ctypes.c_int -ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] - -ssl.EC_KEY_free.restype = None -ssl.EC_KEY_free.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int] - -ssl.EC_KEY_get0_group.restype = ctypes.c_void_p -ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p -ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_set_private_key.restype = ctypes.c_int -ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_KEY_set_conv_form.restype = None -ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int] - -ssl.EC_KEY_set_public_key.restype = ctypes.c_int -ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.i2o_ECPublicKey.restype = ctypes.c_void_p -ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_POINT_new.restype = ctypes.c_void_p -ssl.EC_POINT_new.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_free.restype = None -ssl.EC_POINT_free.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_mul.restype = ctypes.c_int -ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -# this specifies the curve used with ECDSA. -NID_secp256k1 = 714 # from openssl/obj_mac.h +def modinv(a, n): + """Compute the modular inverse of a modulo n + See https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm#Modular_integers. + """ + t1, t2 = 0, 1 + r1, r2 = n, a + while r2 != 0: + q = r1 // r2 + t1, t2 = t2, t1 - q * t2 + r1, r2 = r2, r1 - q * r2 + if r1 > 1: + return None + if t1 < 0: + t1 += n + return t1 + +def jacobi_symbol(n, k): + """Compute the Jacobi symbol of n modulo k + + See http://en.wikipedia.org/wiki/Jacobi_symbol + + For our application k is always prime, so this is the same as the Legendre symbol.""" + assert k > 0 and k & 1, "jacobi symbol is only defined for positive odd k" + n %= k + t = 0 + while n != 0: + while n & 1 == 0: + n >>= 1 + r = k & 7 + t ^= (r == 3 or r == 5) + n, k = k, n + t ^= (n & k & 3 == 3) + n = n % k + if k == 1: + return -1 if t else 1 + return 0 + +def modsqrt(a, p): + """Compute the square root of a modulo p when p % 4 = 3. + + The Tonelli-Shanks algorithm can be used. See https://en.wikipedia.org/wiki/Tonelli-Shanks_algorithm + + Limiting this function to only work for p % 4 = 3 means we don't need to + iterate through the loop. The highest n such that p - 1 = 2^n Q with Q odd + is n = 1. Therefore Q = (p-1)/2 and sqrt = a^((Q+1)/2) = a^((p+1)/4) + + secp256k1's is defined over field of size 2**256 - 2**32 - 977, which is 3 mod 4. + """ + if p % 4 != 3: + raise NotImplementedError("modsqrt only implemented for p % 4 = 3") + sqrt = pow(a, (p + 1)//4, p) + if pow(sqrt, 2, p) == a % p: + return sqrt + return None + +class EllipticCurve: + def __init__(self, p, a, b): + """Initialize elliptic curve y^2 = x^3 + a*x + b over GF(p).""" + self.p = p + self.a = a % p + self.b = b % p + + def affine(self, p1): + """Convert a Jacobian point tuple p1 to affine form, or None if at infinity. + + An affine point is represented as the Jacobian (x, y, 1)""" + x1, y1, z1 = p1 + if z1 == 0: + return None + inv = modinv(z1, self.p) + inv_2 = (inv**2) % self.p + inv_3 = (inv_2 * inv) % self.p + return ((inv_2 * x1) % self.p, (inv_3 * y1) % self.p, 1) + + def negate(self, p1): + """Negate a Jacobian point tuple p1.""" + x1, y1, z1 = p1 + return (x1, (self.p - y1) % self.p, z1) + + def on_curve(self, p1): + """Determine whether a Jacobian tuple p is on the curve (and not infinity)""" + x1, y1, z1 = p1 + z2 = pow(z1, 2, self.p) + z4 = pow(z2, 2, self.p) + return z1 != 0 and (pow(x1, 3, self.p) + self.a * x1 * z4 + self.b * z2 * z4 - pow(y1, 2, self.p)) % self.p == 0 + + def is_x_coord(self, x): + """Test whether x is a valid X coordinate on the curve.""" + x_3 = pow(x, 3, self.p) + return jacobi_symbol(x_3 + self.a * x + self.b, self.p) != -1 + + def lift_x(self, x): + """Given an X coordinate on the curve, return a corresponding affine point.""" + x_3 = pow(x, 3, self.p) + v = x_3 + self.a * x + self.b + y = modsqrt(v, self.p) + if y is None: + return None + return (x, y, 1) + + def double(self, p1): + """Double a Jacobian tuple p1 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Doubling""" + x1, y1, z1 = p1 + if z1 == 0: + return (0, 1, 0) + y1_2 = (y1**2) % self.p + y1_4 = (y1_2**2) % self.p + x1_2 = (x1**2) % self.p + s = (4*x1*y1_2) % self.p + m = 3*x1_2 + if self.a: + m += self.a * pow(z1, 4, self.p) + m = m % self.p + x2 = (m**2 - 2*s) % self.p + y2 = (m*(s - x2) - 8*y1_4) % self.p + z2 = (2*y1*z1) % self.p + return (x2, y2, z2) + + def add_mixed(self, p1, p2): + """Add a Jacobian tuple p1 and an affine tuple p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition (with affine point)""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + assert(z2 == 1) + # Adding to the point at infinity is a no-op + if z1 == 0: + return p2 + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + u2 = (x2 * z1_2) % self.p + s2 = (y2 * z1_3) % self.p + if x1 == u2: + if (y1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - x1 + r = s2 - y1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (x1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - y1*h_3) % self.p + z3 = (h*z1) % self.p + return (x3, y3, z3) + + def add(self, p1, p2): + """Add two Jacobian tuples p1 and p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + # Adding the point at infinity is a no-op + if z1 == 0: + return p2 + if z2 == 0: + return p1 + # Adding an Affine to a Jacobian is more efficient since we save field multiplications and squarings when z = 1 + if z1 == 1: + return self.add_mixed(p2, p1) + if z2 == 1: + return self.add_mixed(p1, p2) + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + z2_2 = (z2**2) % self.p + z2_3 = (z2_2 * z2) % self.p + u1 = (x1 * z2_2) % self.p + u2 = (x2 * z1_2) % self.p + s1 = (y1 * z2_3) % self.p + s2 = (y2 * z1_3) % self.p + if u1 == u2: + if (s1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - u1 + r = s2 - s1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (u1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - s1*h_3) % self.p + z3 = (h*z1*z2) % self.p + return (x3, y3, z3) + + def mul(self, ps): + """Compute a (multi) point multiplication + + ps is a list of (Jacobian tuple, scalar) pairs. + """ + r = (0, 1, 0) + for i in range(255, -1, -1): + r = self.double(r) + for (p, n) in ps: + if ((n >> i) & 1): + r = self.add(r, p) + return r + +SECP256K1 = EllipticCurve(2**256 - 2**32 - 977, 0, 7) +SECP256K1_G = (0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8, 1) SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2 -# Thx to Sam Devlin for the ctypes magic 64-bit fix. -def _check_result(val, func, args): - if val == 0: - raise ValueError - else: - return ctypes.c_void_p (val) - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.errcheck = _check_result - -class CECKey(): - """Wrapper around OpenSSL's EC_KEY""" - - POINT_CONVERSION_COMPRESSED = 2 - POINT_CONVERSION_UNCOMPRESSED = 4 +class ECPubKey(): + """A secp256k1 public key""" def __init__(self): - self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1) - - def __del__(self): - if ssl: - ssl.EC_KEY_free(self.k) - self.k = None - - def set_secretbytes(self, secret): - priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new()) - group = ssl.EC_KEY_get0_group(self.k) - pub_key = ssl.EC_POINT_new(group) - ctx = ssl.BN_CTX_new() - if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx): - raise ValueError("Could not derive public key from the supplied secret.") - ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx) - ssl.EC_KEY_set_private_key(self.k, priv_key) - ssl.EC_KEY_set_public_key(self.k, pub_key) - ssl.EC_POINT_free(pub_key) - ssl.BN_CTX_free(ctx) - return self.k - - def set_privkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def set_pubkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def get_privkey(self): - size = ssl.i2d_ECPrivateKey(self.k, 0) - mb_pri = ctypes.create_string_buffer(size) - ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri))) - return mb_pri.raw - - def get_pubkey(self): - size = ssl.i2o_ECPublicKey(self.k, 0) - mb = ctypes.create_string_buffer(size) - ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb))) - return mb.raw - - def get_raw_ecdh_key(self, other_pubkey): - ecdh_keybuffer = ctypes.create_string_buffer(32) - r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32, - ssl.EC_KEY_get0_public_key(other_pubkey.k), - self.k, 0) - if r != 32: - raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed') - return ecdh_keybuffer.raw - - def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()): - # FIXME: be warned it's not clear what the kdf should be as a default - r = self.get_raw_ecdh_key(other_pubkey) - return kdf(r) - - def sign(self, hash, low_s = True): - # FIXME: need unit tests for below cases - if not isinstance(hash, bytes): - raise TypeError('Hash must be bytes instance; got %r' % hash.__class__) - if len(hash) != 32: - raise ValueError('Hash must be exactly 32 bytes long') - - sig_size0 = ctypes.c_uint32() - sig_size0.value = ssl.ECDSA_size(self.k) - mb_sig = ctypes.create_string_buffer(sig_size0.value) - result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k) - assert 1 == result - assert mb_sig.raw[0] == 0x30 - assert mb_sig.raw[1] == sig_size0.value - 2 - total_size = mb_sig.raw[1] - assert mb_sig.raw[2] == 2 - r_size = mb_sig.raw[3] - assert mb_sig.raw[4 + r_size] == 2 - s_size = mb_sig.raw[5 + r_size] - s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big') - if (not low_s) or s_value <= SECP256K1_ORDER_HALF: - return mb_sig.raw[:sig_size0.value] - else: - low_s_value = SECP256K1_ORDER - s_value - low_s_bytes = (low_s_value).to_bytes(33, byteorder='big') - while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80: - low_s_bytes = low_s_bytes[1:] - new_s_size = len(low_s_bytes) - new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big') - new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big') - return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes - - def verify(self, hash, sig): - """Verify a DER signature""" - return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1 - - def set_compressed(self, compressed): - if compressed: - form = self.POINT_CONVERSION_COMPRESSED + """Construct an uninitialized public key""" + self.valid = False + + def set(self, data): + """Construct a public key from a serialization in compressed or uncompressed format""" + if (len(data) == 65 and data[0] == 0x04): + p = (int.from_bytes(data[1:33], 'big'), int.from_bytes(data[33:65], 'big'), 1) + self.valid = SECP256K1.on_curve(p) + if self.valid: + self.p = p + self.compressed = False + elif (len(data) == 33 and (data[0] == 0x02 or data[0] == 0x03)): + x = int.from_bytes(data[1:33], 'big') + if SECP256K1.is_x_coord(x): + p = SECP256K1.lift_x(x) + # if the oddness of the y co-ord isn't correct, find the other + # valid y + if (p[1] & 1) != (data[0] & 1): + p = SECP256K1.negate(p) + self.p = p + self.valid = True + self.compressed = True + else: + self.valid = False else: - form = self.POINT_CONVERSION_UNCOMPRESSED - ssl.EC_KEY_set_conv_form(self.k, form) - + self.valid = False -class CPubKey(bytes): - """An encapsulated public key - - Attributes: + @property + def is_compressed(self): + return self.compressed - is_valid - Corresponds to CPubKey.IsValid() - is_fullyvalid - Corresponds to CPubKey.IsFullyValid() - is_compressed - Corresponds to CPubKey.IsCompressed() - """ + @property + def is_valid(self): + return self.valid + + def get_bytes(self): + assert(self.valid) + p = SECP256K1.affine(self.p) + if p is None: + return None + if self.compressed: + return bytes([0x02 + (p[1] & 1)]) + p[0].to_bytes(32, 'big') + else: + return bytes([0x04]) + p[0].to_bytes(32, 'big') + p[1].to_bytes(32, 'big') + + def verify_ecdsa(self, sig, msg, low_s=True): + """Verify a strictly DER-encoded ECDSA signature against this pubkey. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA verifier algorithm""" + assert(self.valid) + + # Extract r and s from the DER formatted signature. Return false for + # any DER encoding errors. + if (sig[1] + 2 != len(sig)): + return False + if (len(sig) < 4): + return False + if (sig[0] != 0x30): + return False + if (sig[2] != 0x02): + return False + rlen = sig[3] + if (len(sig) < 6 + rlen): + return False + if rlen < 1 or rlen > 33: + return False + if sig[4] >= 0x80: + return False + if (rlen > 1 and (sig[4] == 0) and not (sig[5] & 0x80)): + return False + r = int.from_bytes(sig[4:4+rlen], 'big') + if (sig[4+rlen] != 0x02): + return False + slen = sig[5+rlen] + if slen < 1 or slen > 33: + return False + if (len(sig) != 6 + rlen + slen): + return False + if sig[6+rlen] >= 0x80: + return False + if (slen > 1 and (sig[6+rlen] == 0) and not (sig[7+rlen] & 0x80)): + return False + s = int.from_bytes(sig[6+rlen:6+rlen+slen], 'big') + + # Verify that r and s are within the group order + if r < 1 or s < 1 or r >= SECP256K1_ORDER or s >= SECP256K1_ORDER: + return False + if low_s and s >= SECP256K1_ORDER_HALF: + return False + z = int.from_bytes(msg, 'big') + + # Run verifier algorithm on r, s + w = modinv(s, SECP256K1_ORDER) + u1 = z*w % SECP256K1_ORDER + u2 = r*w % SECP256K1_ORDER + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, u1), (self.p, u2)])) + if R is None or R[0] != r: + return False + return True + +class ECKey(): + """A secp256k1 private key""" - def __new__(cls, buf, _cec_key=None): - self = super(CPubKey, cls).__new__(cls, buf) - if _cec_key is None: - _cec_key = CECKey() - self._cec_key = _cec_key - self.is_fullyvalid = _cec_key.set_pubkey(self) != 0 - return self + def __init__(self): + self.valid = False + + def set(self, secret, compressed): + """Construct a private key object with given 32-byte secret and compressed flag.""" + assert(len(secret) == 32) + secret = int.from_bytes(secret, 'big') + self.valid = (secret > 0 and secret < SECP256K1_ORDER) + if self.valid: + self.secret = secret + self.compressed = compressed + + def generate(self, compressed=True): + """Generate a random private key (compressed or uncompressed).""" + self.set(random.randrange(1, SECP256K1_ORDER).to_bytes(32, 'big'), compressed) + + def get_bytes(self): + """Retrieve the 32-byte representation of this key.""" + assert(self.valid) + return self.secret.to_bytes(32, 'big') @property def is_valid(self): - return len(self) > 0 + return self.valid @property def is_compressed(self): - return len(self) == 33 - - def verify(self, hash, sig): - return self._cec_key.verify(hash, sig) - - def __str__(self): - return repr(self) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__()) + return self.compressed + def get_pubkey(self): + """Compute an ECPubKey object for this secret key.""" + assert(self.valid) + ret = ECPubKey() + p = SECP256K1.mul([(SECP256K1_G, self.secret)]) + ret.p = p + ret.valid = True + ret.compressed = self.compressed + return ret + + def sign_ecdsa(self, msg, low_s=True): + """Construct a DER-encoded ECDSA signature with this key. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA signer algorithm.""" + assert(self.valid) + z = int.from_bytes(msg, 'big') + # Note: no RFC6979, but a simple random nonce (some tests rely on distinct transactions for the same operation) + k = random.randrange(1, SECP256K1_ORDER) + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, k)])) + r = R[0] % SECP256K1_ORDER + s = (modinv(k, SECP256K1_ORDER) * (z + self.secret * r)) % SECP256K1_ORDER + if low_s and s > SECP256K1_ORDER_HALF: + s = SECP256K1_ORDER - s + # Represent in DER format. The byte representations of r and s have + # length rounded up (255 bits becomes 32 bytes and 256 bits becomes 33 + # bytes). + rb = r.to_bytes((r.bit_length() + 8) // 8, 'big') + sb = s.to_bytes((s.bit_length() + 8) // 8, 'big') + return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index c72cb8835c..7cf51d9223 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -28,7 +28,7 @@ import struct import time from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str +from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -181,7 +181,7 @@ def FromHex(obj, hex_string): # Convert a binary-serializable object to hex (eg for submission via RPC) def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) + return obj.serialize().hex() # Objects that map to bitcoind objects, which can be serialized/deserialized @@ -319,7 +319,7 @@ class CTxIn: def __repr__(self): return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + % (repr(self.prevout), self.scriptSig.hex(), self.nSequence) @@ -343,7 +343,7 @@ class CTxOut: def __repr__(self): return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) + self.scriptPubKey.hex()) class CScriptWitness: @@ -355,7 +355,7 @@ class CScriptWitness: def __repr__(self): return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) + (",".join([x.hex() for x in self.stack])) def is_null(self): if self.stack: @@ -450,6 +450,8 @@ class CTransaction: if flags != 0: self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] self.wit.deserialize(f) + else: + self.wit = CTxWitness() self.nLockTime = struct.unpack("<I", f.read(4))[0] self.sha256 = None self.hash = None @@ -589,6 +591,8 @@ class CBlockHeader: % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, time.ctime(self.nTime), self.nBits, self.nNonce) +BLOCK_HEADER_SIZE = len(CBlockHeader().serialize()) +assert_equal(BLOCK_HEADER_SIZE, 80) class CBlock(CBlockHeader): __slots__ = ("vtx",) @@ -764,7 +768,7 @@ class HeaderAndShortIDs: self.prefilled_txn = [] self.use_witness = False - if p2pheaders_and_shortids != None: + if p2pheaders_and_shortids is not None: self.header = p2pheaders_and_shortids.header self.nonce = p2pheaders_and_shortids.nonce self.shortids = p2pheaders_and_shortids.shortids @@ -822,7 +826,7 @@ class BlockTransactionsRequest: def __init__(self, blockhash=0, indexes = None): self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] + self.indexes = indexes if indexes is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -863,7 +867,7 @@ class BlockTransactions: def __init__(self, blockhash=0, transactions = None): self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] + self.transactions = transactions if transactions is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -1052,7 +1056,7 @@ class msg_getdata: command = b"getdata" def __init__(self, inv=None): - self.inv = inv if inv != None else [] + self.inv = inv if inv is not None else [] def deserialize(self, f): self.inv = deser_vector(f, CInv) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ca5734d67d..7a063ac526 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -118,7 +118,7 @@ class P2PConnection(asyncio.Protocol): # The initial message to send after the connection was made: self.on_connection_send_msg = None self.recvbuf = b"" - self.network = net + self.magic_bytes = MAGIC_BYTES[net] logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) loop = NetworkThread.network_event_loop @@ -170,8 +170,8 @@ class P2PConnection(asyncio.Protocol): while True: if len(self.recvbuf) < 4: return - if self.recvbuf[:4] != MAGIC_BYTES[self.network]: - raise ValueError("got garbage %s" % repr(self.recvbuf)) + if self.recvbuf[:4] != self.magic_bytes: + raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] @@ -218,10 +218,7 @@ class P2PConnection(asyncio.Protocol): def maybe_write(): if not self._transport: return - # Python <3.4.4 does not have is_closing, so we have to check for - # its existence explicitly as long as Bitcoin Core supports all - # Python 3.4 versions. - if hasattr(self._transport, 'is_closing') and self._transport.is_closing(): + if self._transport.is_closing(): return self._transport.write(raw_message_bytes) NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write) @@ -232,7 +229,7 @@ class P2PConnection(asyncio.Protocol): """Build a serialized P2P message""" command = message.command data = message.serialize() - tmsg = MAGIC_BYTES[self.network] + tmsg = self.magic_bytes tmsg += command tmsg += b"\x00" * (12 - len(command)) tmsg += struct.pack("<I", len(data)) diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 45c182f630..c98424e8e2 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. @@ -12,7 +12,7 @@ import socket import struct import array import os -from binascii import unhexlify, hexlify +from binascii import unhexlify # STATE_ESTABLISHED = '01' # STATE_SYN_SENT = '02' @@ -129,17 +129,17 @@ def addr_to_hex(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr = sub[0] + ([0] * nullbytes) + sub[1] else: raise ValueError('Could not parse address %s' % addr) - return hexlify(bytearray(addr)).decode('ascii') + return bytearray(addr).hex() def test_ipv6_local(): ''' diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 2fe44010ba..384062b808 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as SignatureHash(). @@ -9,7 +9,6 @@ This file is modified from python-bitcoinlib. from .messages import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string -from binascii import hexlify import hashlib import struct @@ -385,6 +384,22 @@ class CScriptNum: r[-1] |= 0x80 return bytes([len(r)]) + r + @staticmethod + def decode(vch): + result = 0 + # We assume valid push_size and minimal encoding + value = vch[1:] + if len(value) == 0: + return result + for i, byte in enumerate(value): + result |= int(byte) << 8*i + if value[-1] >= 0x80: + # Mask for all but the highest result bit + num_mask = (2**(len(value)*8) - 1) >> 1 + result &= num_mask + result *= -1 + return result + class CScript(bytes): """Serialized script @@ -525,7 +540,7 @@ class CScript(bytes): def __repr__(self): def _repr(o): if isinstance(o, bytes): - return "x('%s')" % hexlify(o).decode('ascii') + return "x('%s')" % o.hex() else: return repr(o) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index a21c864e75..799b1c74b8 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" @@ -144,7 +144,7 @@ class Socks5Server(): thread.start() def start(self): - assert(not self.running) + assert not self.running self.running = True self.thread = threading.Thread(None, self.run) self.thread.daemon = True diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 44fc185e6d..4aeff24d12 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -29,11 +29,11 @@ from .util import ( get_datadir_path, initialize_datadir, p2p_port, - set_node_times, sync_blocks, sync_mempools, ) + class TestStatus(Enum): PASSED = 1 FAILED = 2 @@ -43,6 +43,8 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 +TMPDIR_PREFIX = "bitcoin_func_test_" + class SkipTest(Exception): """This exception is raised to skip a test""" @@ -92,8 +94,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.setup_clean_chain = False self.nodes = [] self.network_thread = None - self.mocktime = 0 - self.rpc_timewait = 60 # Wait for up to 60 seconds for the RPC server to respond + self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = False self.bind_to_localhost_only = True self.set_test_params() @@ -126,6 +127,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use bitcoin-cli instead of RPC for all commands") + parser.add_argument("--perf", dest="perf", default=False, action="store_true", + help="profile running nodes with perf for the duration of the test") self.add_options(parser) self.options = parser.parse_args() @@ -137,6 +140,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) + self.config = config self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) @@ -151,7 +155,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: - self.options.tmpdir = tempfile.mkdtemp(prefix="test") + self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX) self._start_logging() self.log.debug('Setting up network thread') @@ -199,11 +203,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): node.cleanup_on_exit = False self.log.info("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: + should_clean_up = ( + not self.options.nocleanup and + not self.options.noshutdown and + success != TestStatus.FAILED and + not self.options.perf + ) + if should_clean_up: self.log.info("Cleaning up {} on exit".format(self.options.tmpdir)) cleanup_tree_on_exit = True + elif self.options.perf: + self.log.warning("Not cleaning up dir {} due to perf data".format(self.options.tmpdir)) + cleanup_tree_on_exit = False else: - self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) + self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False if success == TestStatus.PASSED: @@ -261,6 +274,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.add_nodes(self.num_nodes, extra_args) self.start_nodes() self.import_deterministic_coinbase_privkeys() + if not self.setup_clean_chain: + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) + # To ensure that all nodes are out of IBD, the most recent block + # must have a timestamp not too old (see IsInitialBlockDownload()). + self.log.debug('Generate a block with current time') + block_hash = self.nodes[0].generate(1)[0] + block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) + for n in self.nodes: + n.submitblock(block) + chain_info = n.getblockchaininfo() + assert_equal(chain_info["blocks"], 200) + assert_equal(chain_info["initialblockdownload"], False) def import_deterministic_coinbase_privkeys(self): for n in self.nodes: @@ -279,7 +305,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Public helper methods. These can be accessed by the subclass test scripts. def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None): - """Instantiate TestNode objects""" + """Instantiate TestNode objects. + + Should only be called once after the nodes have been specified in + set_test_params().""" if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -292,7 +321,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, get_datadir_path(self.options.tmpdir, i), rpchost=rpchost, timewait=self.rpc_timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli)) + self.nodes.append(TestNode( + i, + get_datadir_path(self.options.tmpdir, i), + rpchost=rpchost, + timewait=self.rpc_timeout, + bitcoind=binary[i], + bitcoin_cli=self.options.bitcoincli, + coverage_dir=self.options.coveragedir, + cwd=self.options.tmpdir, + extra_conf=extra_confs[i], + extra_args=extra_args[i], + use_cli=self.options.usecli, + start_perf=self.options.perf, + )) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" @@ -325,16 +367,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) - def stop_node(self, i, expected_stderr=''): + def stop_node(self, i, expected_stderr='', wait=0): """Stop a bitcoind test node""" - self.nodes[i].stop_node(expected_stderr) + self.nodes[i].stop_node(expected_stderr, wait=wait) self.nodes[i].wait_until_stopped() - def stop_nodes(self): + def stop_nodes(self, wait=0): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes - node.stop_node() + node.stop_node(wait=wait) for node in self.nodes: # Wait for nodes to stop @@ -354,7 +396,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """ disconnect_nodes(self.nodes[1], 2) disconnect_nodes(self.nodes[2], 1) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) def join_network(self): """ @@ -363,28 +406,15 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): connect_nodes_bi(self.nodes, 1, 2) self.sync_all() - def sync_all(self, node_groups=None): - if not node_groups: - node_groups = [self.nodes] + def sync_blocks(self, nodes=None, **kwargs): + sync_blocks(nodes or self.nodes, **kwargs) - for group in node_groups: - sync_blocks(group) - sync_mempools(group) + def sync_mempools(self, nodes=None, **kwargs): + sync_mempools(nodes or self.nodes, **kwargs) - def enable_mocktime(self): - """Enable mocktime for the script. - - mocktime may be needed for scripts that use the cached version of the - blockchain. If the cached version of the blockchain is used without - mocktime then the mempools will not sync due to IBD. - - For backward compatibility of the python scripts with previous - versions of the cache, this helper function sets mocktime to Jan 1, - 2014 + (201 * 10 * 60)""" - self.mocktime = 1388534400 + (201 * 10 * 60) - - def disable_mocktime(self): - self.mocktime = 0 + def sync_all(self, nodes=None, **kwargs): + self.sync_blocks(nodes, **kwargs) + self.sync_mempools(nodes, **kwargs) # Private helper methods. These should not be accessed by the subclass test scripts. @@ -419,7 +449,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. - Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Create a cache of a 199-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" assert self.num_nodes <= MAX_NODES @@ -443,7 +473,18 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): args = [self.options.bitcoind, "-datadir=" + datadir, '-disablewallet'] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) - self.nodes.append(TestNode(i, get_datadir_path(self.options.cachedir, i), extra_conf=["bind=127.0.0.1"], extra_args=[], rpchost=None, timewait=self.rpc_timewait, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=None)) + self.nodes.append(TestNode( + i, + get_datadir_path(self.options.cachedir, i), + extra_conf=["bind=127.0.0.1"], + extra_args=[], + rpchost=None, + timewait=self.rpc_timeout, + bitcoind=self.options.bitcoind, + bitcoin_cli=self.options.bitcoincli, + coverage_dir=None, + cwd=self.options.tmpdir, + )) self.nodes[i].args = args self.start_node(i) @@ -451,28 +492,22 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: node.wait_for_rpc_connection() - # Create a 200-block-long chain; each of the 4 first nodes + # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. - # Note: To preserve compatibility with older versions of - # initialize_chain, only 4 nodes will generate coins. - # - # blocks are created with timestamps 10 minutes apart - # starting from 2010 minutes in the past - self.enable_mocktime() - block_time = self.mocktime - (201 * 10 * 60) - for i in range(2): - for peer in range(4): - for j in range(25): - set_node_times(self.nodes, block_time) - self.nodes[peer].generatetoaddress(1, self.nodes[peer].get_deterministic_priv_key().address) - block_time += 10 * 60 - # Must sync before next peer starts generating blocks - sync_blocks(self.nodes) + # The 4th node gets only 24 immature blocks so that the very last + # block in the cache does not age too much (have an old tip age). + # This is needed so that we are out of IBD when the test starts, + # see the tip age check in IsInitialBlockDownload(). + for i in range(8): + self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address) + self.sync_blocks() + + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] - self.disable_mocktime() def cache_path(n, *paths): return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 27f99c259c..8b2006a05c 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -18,6 +18,8 @@ import tempfile import time import urllib.parse import collections +import shlex +import sys from .authproxy import JSONRPCException from .util import ( @@ -29,9 +31,6 @@ from .util import ( p2p_port, ) -# For Python 3.4 compatibility -JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) - BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -59,7 +58,13 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False): + def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + """ + Kwargs: + start_perf (bool): If True, begin profiling the node with `perf` as soon as + the node starts. + """ + self.index = i self.datadir = datadir self.stdout_dir = os.path.join(self.datadir, "stdout") @@ -68,7 +73,8 @@ class TestNode(): self.rpc_timeout = timewait self.binary = bitcoind self.coverage_dir = coverage_dir - if extra_conf != None: + self.cwd = cwd + if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. # For those callers that need more flexibility, they can just set the args property directly. @@ -81,12 +87,12 @@ class TestNode(): "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", - "-mocktime=" + str(mocktime), - "-uacomment=testnode%d" % i + "-uacomment=testnode%d" % i, ] self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli + self.start_perf = start_perf self.running = False self.process = None @@ -95,6 +101,8 @@ class TestNode(): self.url = None self.log = logging.getLogger('TestFramework.node%d' % i) self.cleanup_on_exit = True # Whether to kill the node when this object goes away + # Cache perf subprocesses here by their data output filename. + self.perf_subprocesses = {} self.p2ps = [] @@ -160,7 +168,7 @@ class TestNode(): assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") return getattr(self.rpc, name) - def start(self, extra_args=None, *, stdout=None, stderr=None, **kwargs): + def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args @@ -173,6 +181,9 @@ class TestNode(): self.stderr = stderr self.stdout = stdout + if cwd is None: + cwd = self.cwd + # Delete any existing cookie file -- if such a file exists (eg due to # unclean shutdown), it will get overwritten anyway by bitcoind, and # potentially interfere with our attempt to authenticate @@ -181,11 +192,14 @@ class TestNode(): # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") - self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, **kwargs) + self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") + if self.start_perf: + self._start_perf() + def wait_for_rpc_connection(self): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second @@ -195,12 +209,15 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) - self.rpc.getblockcount() + rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up + self.log.debug("RPC successfully started") + if self.use_cli: + return + self.rpc = rpc self.rpc_connected = True self.url = self.rpc.url - self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? @@ -228,16 +245,20 @@ class TestNode(): wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name)) return self.rpc / wallet_path - def stop_node(self, expected_stderr=''): + def stop_node(self, expected_stderr='', wait=0): """Stop the node.""" if not self.running: return self.log.debug("Stopping node") try: - self.stop() + self.stop(wait=wait) except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") + # If there are any running perf processes, stop them. + for profile_name in tuple(self.perf_subprocesses.keys()): + self._stop_perf(profile_name) + # Check that stderr is as expected self.stderr.seek(0) stderr = self.stderr.read().decode('utf-8').strip() @@ -317,6 +338,84 @@ class TestNode(): increase_allowed * 100, before_memory_usage, after_memory_usage, perc_increase_memory_usage * 100)) + @contextlib.contextmanager + def profile_with_perf(self, profile_name): + """ + Context manager that allows easy profiling of node activity using `perf`. + + See `test/functional/README.md` for details on perf usage. + + Args: + profile_name (str): This string will be appended to the + profile data filename generated by perf. + """ + subp = self._start_perf(profile_name) + + yield + + if subp: + self._stop_perf(profile_name) + + def _start_perf(self, profile_name=None): + """Start a perf process to profile this node. + + Returns the subprocess running perf.""" + subp = None + + def test_success(cmd): + return subprocess.call( + # shell=True required for pipe use below + cmd, shell=True, + stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0 + + if not sys.platform.startswith('linux'): + self.log.warning("Can't profile with perf; only available on Linux platforms") + return None + + if not test_success('which perf'): + self.log.warning("Can't profile with perf; must install perf-tools") + return None + + if not test_success('readelf -S {} | grep .debug_str'.format(shlex.quote(self.binary))): + self.log.warning( + "perf output won't be very useful without debug symbols compiled into bitcoind") + + output_path = tempfile.NamedTemporaryFile( + dir=self.datadir, + prefix="{}.perf.data.".format(profile_name or 'test'), + delete=False, + ).name + + cmd = [ + 'perf', 'record', + '-g', # Record the callgraph. + '--call-graph', 'dwarf', # Compatibility for gcc's --fomit-frame-pointer. + '-F', '101', # Sampling frequency in Hz. + '-p', str(self.process.pid), + '-o', output_path, + ] + subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.perf_subprocesses[profile_name] = subp + + return subp + + def _stop_perf(self, profile_name): + """Stop (and pop) a perf subprocess.""" + subp = self.perf_subprocesses.pop(profile_name) + output_path = subp.args[subp.args.index('-o') + 1] + + subp.terminate() + subp.wait(timeout=10) + + stderr = subp.stderr.read().decode() + if 'Consider tweaking /proc/sys/kernel/perf_event_paranoid' in stderr: + self.log.warning( + "perf couldn't collect data! Try " + "'sudo sysctl -w kernel.perf_event_paranoid=-1'") + else: + report_cmd = "perf report -i {}".format(output_path) + self.log.info("See perf output by running '{}'".format(report_cmd)) + def assert_start_raises_init_error(self, extra_args=None, expected_msg=None, match=ErrorMatch.FULL_TEXT, *args, **kwargs): """Attempt to start the node and expect it to raise an error. @@ -402,6 +501,14 @@ class TestNodeCLIAttr: def get_request(self, *args, **kwargs): return lambda: self(*args, **kwargs) +def arg_to_cli(arg): + if isinstance(arg, bool): + return str(arg).lower() + elif isinstance(arg, dict) or isinstance(arg, list): + return json.dumps(arg) + else: + return str(arg) + class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" @@ -433,8 +540,8 @@ class TestNodeCLI(): def send_cli(self, command=None, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" - pos_args = [str(arg).lower() if type(arg) is bool else str(arg) for arg in args] - named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()] + pos_args = [arg_to_cli(arg) for arg in args] + named_args = [str(key) + "=" + arg_to_cli(value) for (key, value) in kwargs.items()] assert not (pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call" p_args = [self.binary, "-datadir=" + self.datadir] + self.options if named_args: @@ -455,5 +562,5 @@ class TestNodeCLI(): raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except JSONDecodeError: + except json.JSONDecodeError: return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index b355816d8b..190301b215 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" from base64 import b64encode -from binascii import hexlify, unhexlify +from binascii import unhexlify from decimal import Decimal, ROUND_DOWN import hashlib import inspect @@ -182,9 +182,6 @@ def check_json_precision(): def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def bytes_to_hex_str(byte_str): - return hexlify(byte_str).decode('ascii') - def hash256(byte_str): sha256 = hashlib.sha256() sha256.update(byte_str) @@ -267,7 +264,7 @@ def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): - assert(n <= MAX_NODES) + assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): @@ -326,12 +323,14 @@ def get_auth_cookie(datadir): if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") - if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): + try: with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] password = split_userpass[1] + except OSError: + pass if user is None or password is None: raise ValueError("No RPC credentials") return user, password @@ -410,12 +409,12 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): # Transaction/Block functions ############################# -def find_output(node, txid, amount): +def find_output(node, txid, amount, *, blockhash=None): """ Return index to output of txid with value amount Raises exception if there is none. """ - txdata = node.getrawtransaction(txid, 1) + txdata = node.getrawtransaction(txid, 1, blockhash) for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i @@ -425,7 +424,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ - assert(confirmations_required >= 0) + assert confirmations_required >= 0 utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] @@ -470,7 +469,7 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransactionwithwallet(rawtx) - txid = from_node.sendrawtransaction(signresult["hex"], True) + txid = from_node.sendrawtransaction(signresult["hex"], 0) return (txid, signresult["hex"], fee) @@ -503,7 +502,7 @@ def create_confirmed_utxos(fee, node, count): node.generate(1) utxos = node.listunspent() - assert(len(utxos) >= count) + assert len(utxos) >= count return utxos # Create large OP_RETURN txouts that can be appended to a transaction @@ -542,7 +541,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): newtx = newtx + txouts newtx = newtx + rawtx[94:] signresult = node.signrawtransactionwithwallet(newtx, None, "NONE") - txid = node.sendrawtransaction(signresult["hex"], True) + txid = node.sendrawtransaction(signresult["hex"], 0) txids.append(txid) return txids diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py new file mode 100755 index 0000000000..c0dfa4c3f0 --- /dev/null +++ b/test/functional/test_framework/wallet_util.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Useful util functions for testing the wallet""" +from collections import namedtuple + +from test_framework.address import ( + key_to_p2pkh, + key_to_p2sh_p2wpkh, + key_to_p2wpkh, + script_to_p2sh, + script_to_p2sh_p2wsh, + script_to_p2wsh, +) +from test_framework.script import ( + CScript, + OP_0, + OP_2, + OP_3, + OP_CHECKMULTISIG, + OP_CHECKSIG, + OP_DUP, + OP_EQUAL, + OP_EQUALVERIFY, + OP_HASH160, + hash160, + sha256, +) +from test_framework.util import hex_str_to_bytes + +Key = namedtuple('Key', ['privkey', + 'pubkey', + 'p2pkh_script', + 'p2pkh_addr', + 'p2wpkh_script', + 'p2wpkh_addr', + 'p2sh_p2wpkh_script', + 'p2sh_p2wpkh_redeem_script', + 'p2sh_p2wpkh_addr']) + +Multisig = namedtuple('Multisig', ['privkeys', + 'pubkeys', + 'p2sh_script', + 'p2sh_addr', + 'redeem_script', + 'p2wsh_script', + 'p2wsh_addr', + 'p2sh_p2wsh_script', + 'p2sh_p2wsh_addr']) + +def get_key(node): + """Generate a fresh key on node + + Returns a named tuple of privkey, pubkey and all address and scripts.""" + addr = node.getnewaddress() + pubkey = node.getaddressinfo(addr)['pubkey'] + pkh = hash160(hex_str_to_bytes(pubkey)) + return Key(privkey=node.dumpprivkey(addr), + pubkey=pubkey, + p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_addr=key_to_p2pkh(pubkey), + p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_addr=key_to_p2wpkh(pubkey), + p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), + p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) + +def get_multisig(node): + """Generate a fresh 2-of-3 multisig on node + + Returns a named tuple of privkeys, pubkeys and all address and scripts.""" + addrs = [] + pubkeys = [] + for _ in range(3): + addr = node.getaddressinfo(node.getnewaddress()) + addrs.append(addr['address']) + pubkeys.append(addr['pubkey']) + script_code = CScript([OP_2] + [hex_str_to_bytes(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) + witness_script = CScript([OP_0, sha256(script_code)]) + return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs], + pubkeys=pubkeys, + p2sh_script=CScript([OP_HASH160, hash160(script_code), OP_EQUAL]).hex(), + p2sh_addr=script_to_p2sh(script_code), + redeem_script=script_code.hex(), + p2wsh_script=witness_script.hex(), + p2wsh_addr=script_to_p2wsh(script_code), + p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(), + p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code)) + +def test_address(node, address, **kwargs): + """Get address info for `address` and test whether the returned values are as expected.""" + addr_info = node.getaddressinfo(address) + for key, value in kwargs.items(): + if value is None: + if key in addr_info.keys(): + raise AssertionError("key {} unexpectedly returned in getaddressinfo.".format(key)) + elif addr_info[key] != value: + raise AssertionError("key {} value {} did not match expected value {}".format(key, addr_info[key], value)) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index da55a3a156..86f334e942 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -7,8 +7,6 @@ This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. -Functional tests are disabled on Windows by default. Use --force to run them anyway. - For a description of arguments recognized by test scripts, see `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -108,17 +106,19 @@ BASE_SCRIPTS = [ 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', + 'tool_wallet.py', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', + 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'mempool_reorg.py', 'mempool_persist.py', 'wallet_multiwallet.py', 'wallet_multiwallet.py --usecli', - 'wallet_disableprivatekeys.py', - 'wallet_disableprivatekeys.py --usecli', + 'wallet_createwallet.py', + 'wallet_createwallet.py --usecli', 'interface_http.py', 'interface_rpc.py', 'rpc_psbt.py', @@ -144,6 +144,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', + 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', 'feature_rbf.py', 'mempool_packages.py', @@ -153,6 +154,7 @@ BASE_SCRIPTS = [ 'wallet_importprunedfunds.py', 'p2p_leak_tx.py', 'rpc_signmessage.py', + 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', 'wallet_import_rescan.py', @@ -173,11 +175,15 @@ BASE_SCRIPTS = [ 'wallet_fallbackfee.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', + 'wallet_create_tx.py', 'p2p_fingerprint.py', 'feature_uacomment.py', + 'wallet_coinbase_category.py', 'feature_filelock.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', + 'rpc_deriveaddresses.py', + 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', @@ -185,6 +191,7 @@ BASE_SCRIPTS = [ 'feature_config_args.py', 'rpc_help.py', 'feature_help.py', + 'feature_shutdown.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -219,7 +226,6 @@ def main(): parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') - parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') @@ -246,22 +252,12 @@ def main(): # Create base test directory tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) - # If we fixed the command-line and filename encoding issue on Windows, these two lines could be removed - if config["environment"]["EXEEXT"] == ".exe": - tmpdir = "%s/test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) - os.makedirs(tmpdir) logging.debug("Temporary test directory at %s" % tmpdir) enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - if config["environment"]["EXEEXT"] == ".exe" and not args.force: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Tests currently disabled on Windows by default. Use --force option to enable") - sys.exit(0) - if not enable_bitcoind: print("No functional tests to run.") print("Rerun ./configure with --with-daemon and then make") @@ -272,7 +268,7 @@ def main(): if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. - tests = [re.sub("\.py$", "", test) + ".py" for test in tests] + tests = [test + ".py" if ".py" not in test else test for test in tests] for test in tests: if test in ALL_SCRIPTS: test_list.append(test) @@ -483,6 +479,11 @@ class TestHandler: log_stderr)) if not self.jobs: raise IndexError('pop from empty list') + + # Print remaining running jobs when all jobs have been started. + if not self.test_list: + print("Remaining jobs: [{}]".format(", ".join(j[0] for j in self.jobs))) + dot_count = 0 while True: # Return first proc that finishes @@ -558,7 +559,7 @@ class TestResult(): def check_script_prefixes(): """Check that test scripts start with one of the allowed name prefixes.""" - good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet)_") + good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] if bad_script_names: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py new file mode 100755 index 0000000000..fbcf21e729 --- /dev/null +++ b/test/functional/tool_wallet.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoin-wallet.""" +import subprocess +import textwrap + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class ToolWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def bitcoin_wallet_process(self, *args): + binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] + args = ['-datadir={}'.format(self.nodes[0].datadir), '-regtest'] + list(args) + return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + def assert_raises_tool_error(self, error, *args): + p = self.bitcoin_wallet_process(*args) + stdout, stderr = p.communicate() + assert_equal(p.poll(), 1) + assert_equal(stdout, '') + assert_equal(stderr.strip(), error) + + def assert_tool_output(self, output, *args): + p = self.bitcoin_wallet_process(*args) + stdout, stderr = p.communicate() + assert_equal(p.poll(), 0) + assert_equal(stderr, '') + assert_equal(stdout, output) + + def run_test(self): + + self.assert_raises_tool_error('Invalid command: foo', 'foo') + # `bitcoin-wallet help` is an error. Use `bitcoin-wallet -help` + self.assert_raises_tool_error('Invalid command: help', 'help') + self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info') + self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') + + # stop the node to close the wallet to call info command + self.stop_node(0) + + out = textwrap.dedent('''\ + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2 + Transactions: 0 + Address Book: 3 + ''') + self.assert_tool_output(out, '-wallet=wallet.dat', 'info') + + # mutate the wallet to check the info command output changes accordingly + self.start_node(0) + self.nodes[0].generate(1) + self.stop_node(0) + + out = textwrap.dedent('''\ + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2 + Transactions: 1 + Address Book: 3 + ''') + self.assert_tool_output(out, '-wallet=wallet.dat', 'info') + + out = textwrap.dedent('''\ + Topping up keypool... + Wallet info + =========== + Encrypted: no + HD (hd seed available): yes + Keypool Size: 2000 + Transactions: 0 + Address Book: 0 + ''') + self.assert_tool_output(out, '-wallet=foo', 'create') + + self.start_node(0, ['-wallet=foo']) + out = self.nodes[0].getwalletinfo() + self.stop_node(0) + + assert_equal(0, out['txcount']) + assert_equal(1000, out['keypoolsize']) + assert_equal(1000, out['keypoolsize_hd_internal']) + assert_equal(True, 'hdseedid' in out) + +if __name__ == '__main__': + ToolWalletTest().main() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index e5ac2c8bd4..e86679bc31 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. @@ -13,7 +13,13 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, +) + class AbandonConflictTest(BitcoinTestFramework): def set_test_params(self): @@ -25,12 +31,12 @@ class AbandonConflictTest(BitcoinTestFramework): def run_test(self): self.nodes[1].generate(100) - sync_blocks(self.nodes) + self.sync_blocks() balance = self.nodes[0].getbalance() txA = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txB = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txC = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[1].generate(1) # Can not abandon non-wallet transaction @@ -38,23 +44,23 @@ class AbandonConflictTest(BitcoinTestFramework): # Can not abandon confirmed transaction assert_raises_rpc_error(-5, 'Transaction not eligible for abandonment', lambda: self.nodes[0].abandontransaction(txid=txA)) - sync_blocks(self.nodes) + self.sync_blocks() newbalance = self.nodes[0].getbalance() - assert(balance - newbalance < Decimal("0.001")) #no more than fees lost + assert balance - newbalance < Decimal("0.001") #no more than fees lost balance = newbalance # Disconnect nodes so node0's transactions don't get into node1's mempool disconnect_nodes(self.nodes[0], 1) # Identify the 10btc outputs - nA = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txA, 1)["vout"]) if vout["value"] == Decimal("10")) - nB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txB, 1)["vout"]) if vout["value"] == Decimal("10")) - nC = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txC, 1)["vout"]) if vout["value"] == Decimal("10")) + nA = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txA)["details"] if tx_out["amount"] == Decimal("10")) + nB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txB)["details"] if tx_out["amount"] == Decimal("10")) + nC = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txC)["details"] if tx_out["amount"] == Decimal("10")) - inputs =[] + inputs = [] # spend 10btc outputs from txA and txB - inputs.append({"txid":txA, "vout":nA}) - inputs.append({"txid":txB, "vout":nB}) + inputs.append({"txid": txA, "vout": nA}) + inputs.append({"txid": txB, "vout": nB}) outputs = {} outputs[self.nodes[0].getnewaddress()] = Decimal("14.99998") @@ -63,12 +69,12 @@ class AbandonConflictTest(BitcoinTestFramework): txAB1 = self.nodes[0].sendrawtransaction(signed["hex"]) # Identify the 14.99998btc output - nAB = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txAB1, 1)["vout"]) if vout["value"] == Decimal("14.99998")) + nAB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txAB1)["details"] if tx_out["amount"] == Decimal("14.99998")) #Create a child tx spending AB1 and C inputs = [] - inputs.append({"txid":txAB1, "vout":nAB}) - inputs.append({"txid":txC, "vout":nC}) + inputs.append({"txid": txAB1, "vout": nAB}) + inputs.append({"txid": txC, "vout": nC}) outputs = {} outputs[self.nodes[0].getnewaddress()] = Decimal("24.9996") signed2 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) @@ -76,8 +82,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Create a child tx spending ABC2 signed3_change = Decimal("24.999") - inputs = [ {"txid":txABC2, "vout":0} ] - outputs = { self.nodes[0].getnewaddress(): signed3_change } + inputs = [{"txid": txABC2, "vout": 0}] + outputs = {self.nodes[0].getnewaddress(): signed3_change} signed3 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) # note tx is never directly referenced, only abandoned as a child of the above self.nodes[0].sendrawtransaction(signed3["hex"]) @@ -105,7 +111,7 @@ class AbandonConflictTest(BitcoinTestFramework): unconfbalance = self.nodes[0].getunconfirmedbalance() + self.nodes[0].getbalance() assert_equal(unconfbalance, newbalance) # Also shouldn't show up in listunspent - assert(not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)]) + assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)] balance = newbalance # Abandon original transaction and verify inputs are available again @@ -145,8 +151,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Create a double spend of AB1 by spending again from only A's 10 output # Mine double spend from node 1 - inputs =[] - inputs.append({"txid":txA, "vout":nA}) + inputs = [] + inputs.append({"txid": txA, "vout": nA}) outputs = {} outputs[self.nodes[1].getnewaddress()] = Decimal("9.9999") tx = self.nodes[0].createrawtransaction(inputs, outputs) @@ -155,7 +161,7 @@ class AbandonConflictTest(BitcoinTestFramework): self.nodes[1].generate(1) connect_nodes(self.nodes[0], 1) - sync_blocks(self.nodes) + self.sync_blocks() # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted newbalance = self.nodes[0].getbalance() @@ -172,5 +178,6 @@ class AbandonConflictTest(BitcoinTestFramework): self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") self.log.info(str(balance) + " -> " + str(newbalance) + " ?") + if __name__ == '__main__': AbandonConflictTest().main() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 0f75045c9d..a40613dfc7 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -54,13 +54,15 @@ from decimal import Decimal import itertools from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import ( + descsum_create, + descsum_check, +) from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, - sync_blocks, - sync_mempools, ) @@ -98,53 +100,103 @@ class AddressTypeTest(BitcoinTestFramework): def test_address(self, node, address, multisig, typ): """Run sanity checks on an address.""" info = self.nodes[node].getaddressinfo(address) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] + assert_equal(info.get('solvable'), True) + if not multisig and typ == 'legacy': # P2PKH - assert(not info['isscript']) - assert(not info['iswitness']) - assert('pubkey' in info) + assert not info['isscript'] + assert not info['iswitness'] + assert 'pubkey' in info elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert(info['isscript']) - assert(not info['iswitness']) + assert info['isscript'] + assert not info['iswitness'] assert_equal(info['script'], 'witness_v0_keyhash') - assert('pubkey' in info) + assert 'pubkey' in info elif not multisig and typ == 'bech32': # P2WPKH - assert(not info['isscript']) - assert(info['iswitness']) + assert not info['isscript'] + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 40) - assert('pubkey' in info) + assert 'pubkey' in info elif typ == 'legacy': # P2SH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(not info['iswitness']) - assert('pubkeys' in info) + assert not info['iswitness'] + assert 'pubkeys' in info elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'witness_v0_scripthash') - assert(not info['iswitness']) - assert(info['embedded']['isscript']) + assert not info['iswitness'] + assert info['embedded']['isscript'] assert_equal(info['embedded']['script'], 'multisig') - assert(info['embedded']['iswitness']) + assert info['embedded']['iswitness'] assert_equal(info['embedded']['witness_version'], 0) assert_equal(len(info['embedded']['witness_program']), 64) - assert('pubkeys' in info['embedded']) + assert 'pubkeys' in info['embedded'] elif typ == 'bech32': # P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(info['iswitness']) + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 64) - assert('pubkeys' in info) + assert 'pubkeys' in info + else: + # Unknown type + assert False + + def test_desc(self, node, address, multisig, typ, utxo): + """Run sanity checks on a descriptor reported by getaddressinfo.""" + info = self.nodes[node].getaddressinfo(address) + assert 'desc' in info + assert_equal(info['desc'], utxo['desc']) + assert self.nodes[node].validateaddress(address)['isvalid'] + + # Use a ridiculously roundabout way to find the key origin info through + # the PSBT logic. However, this does test consistency between the PSBT reported + # fingerprints/paths and the descriptor logic. + psbt = self.nodes[node].createpsbt([{'txid':utxo['txid'], 'vout':utxo['vout']}],[{address:0.00010000}]) + psbt = self.nodes[node].walletprocesspsbt(psbt, False, "ALL", True) + decode = self.nodes[node].decodepsbt(psbt['psbt']) + key_descs = {} + for deriv in decode['inputs'][0]['bip32_derivs']: + assert_equal(len(deriv['master_fingerprint']), 8) + assert_equal(deriv['path'][0], 'm') + key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] + + # Verify the descriptor checksum against the Python implementation + assert descsum_check(info['desc']) + # Verify that stripping the checksum and recreating it using Python roundtrips + assert info['desc'] == descsum_create(info['desc'][:-9]) + # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips + assert info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor'] + + if not multisig and typ == 'legacy': + # P2PKH + assert_equal(info['desc'], descsum_create("pkh(%s)" % key_descs[info['pubkey']])) + elif not multisig and typ == 'p2sh-segwit': + # P2SH-P2WPKH + assert_equal(info['desc'], descsum_create("sh(wpkh(%s))" % key_descs[info['pubkey']])) + elif not multisig and typ == 'bech32': + # P2WPKH + assert_equal(info['desc'], descsum_create("wpkh(%s)" % key_descs[info['pubkey']])) + elif typ == 'legacy': + # P2SH-multisig + assert_equal(info['desc'], descsum_create("sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) + elif typ == 'p2sh-segwit': + # P2SH-P2WSH-multisig + assert_equal(info['desc'], descsum_create("sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]]))) + elif typ == 'bech32': + # P2WSH-multisig + assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) else: # Unknown type - assert(False) + assert False def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) @@ -166,7 +218,7 @@ class AddressTypeTest(BitcoinTestFramework): # Mine 101 blocks on node5 to bring nodes out of IBD and make sure that # no coinbases are maturing for the nodes-under-test during the test self.nodes[5].generate(101) - sync_blocks(self.nodes) + self.sync_blocks() uncompressed_1 = "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee" uncompressed_2 = "047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77" @@ -198,6 +250,7 @@ class AddressTypeTest(BitcoinTestFramework): self.log.debug("Old balances are {}".format(old_balances)) to_send = (old_balances[from_node] / 101).quantize(Decimal("0.00000001")) sends = {} + addresses = {} self.log.debug("Prepare sends") for n, to_node in enumerate(range(from_node, from_node + 4)): @@ -228,10 +281,11 @@ class AddressTypeTest(BitcoinTestFramework): # Output entry sends[address] = to_send * 10 * (1 + n) + addresses[to_node] = (address, typ) self.log.debug("Sending: {}".format(sends)) self.nodes[from_node].sendmany("", sends) - sync_mempools(self.nodes) + self.sync_mempools() unconf_balances = self.get_balances(False) self.log.debug("Check unconfirmed balances: {}".format(unconf_balances)) @@ -242,7 +296,18 @@ class AddressTypeTest(BitcoinTestFramework): # node5 collects fee and block subsidy to keep accounting simple self.nodes[5].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() + + # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor + for n, to_node in enumerate(range(from_node, from_node + 4)): + to_node %= 4 + found = False + for utxo in self.nodes[to_node].listunspent(): + if utxo['address'] == addresses[to_node][0]: + found = True + self.test_desc(to_node, addresses[to_node][0], multisig, addresses[to_node][1], utxo) + break + assert found new_balances = self.get_balances() self.log.debug("Check new balances: {}".format(new_balances)) @@ -261,7 +326,7 @@ class AddressTypeTest(BitcoinTestFramework): # Fund node 4: self.nodes[5].sendtoaddress(self.nodes[4].getnewaddress(), Decimal("1")) self.nodes[5].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[4].getbalance(), 1) self.log.info("Nodes with addresstype=legacy never use a P2WPKH change output") diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 32ec385fa1..55c517e92f 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. @@ -36,7 +36,12 @@ from random import randint import shutil from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, sync_blocks, sync_mempools +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes, +) + class WalletBackupTest(BitcoinTestFramework): def set_test_params(self): @@ -48,7 +53,7 @@ class WalletBackupTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.setup_nodes() connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) @@ -75,9 +80,9 @@ class WalletBackupTest(BitcoinTestFramework): # Have the miner (node3) mine a block. # Must sync mempools before mining. - sync_mempools(self.nodes) + self.sync_mempools() self.nodes[3].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # As above, this mirrors the original bash test. def start_three(self): @@ -102,13 +107,13 @@ class WalletBackupTest(BitcoinTestFramework): def run_test(self): self.log.info("Generating initial blockchain") self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[1].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[2].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() self.nodes[3].generate(100) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) @@ -165,7 +170,7 @@ class WalletBackupTest(BitcoinTestFramework): self.log.info("Re-starting nodes") self.start_three() - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) @@ -189,7 +194,7 @@ class WalletBackupTest(BitcoinTestFramework): self.nodes[1].importwallet(os.path.join(self.nodes[1].datadir, 'wallet.dump')) self.nodes[2].importwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py new file mode 100755 index 0000000000..e2a20beec5 --- /dev/null +++ b/test/functional/wallet_balance.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the wallet balance RPC methods.""" +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +RANDOM_COINBASE_ADDRESS = 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ' + +def create_transactions(node, address, amt, fees): + # Create and sign raw transactions from node to address for amt. + # Creates a transaction for each fee and returns an array + # of the raw transactions. + utxos = node.listunspent(0) + + # Create transactions + inputs = [] + ins_total = 0 + for utxo in utxos: + inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]}) + ins_total += utxo['amount'] + if ins_total > amt: + break + + txs = [] + for fee in fees: + outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee} + raw_tx = node.createrawtransaction(inputs, outputs, 0, True) + raw_tx = node.signrawtransactionwithwallet(raw_tx) + txs.append(raw_tx) + + return txs + +class WalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + # Check that nodes don't own any UTXOs + assert_equal(len(self.nodes[0].listunspent()), 0) + assert_equal(len(self.nodes[1].listunspent()), 0) + + self.log.info("Mining one block for each node") + + self.nodes[0].generate(1) + self.sync_all() + self.nodes[1].generate(1) + self.nodes[1].generatetoaddress(100, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + assert_equal(self.nodes[0].getbalance(), 50) + assert_equal(self.nodes[1].getbalance(), 50) + + self.log.info("Test getbalance with different arguments") + assert_equal(self.nodes[0].getbalance("*"), 50) + assert_equal(self.nodes[0].getbalance("*", 1), 50) + assert_equal(self.nodes[0].getbalance("*", 1, True), 50) + assert_equal(self.nodes[0].getbalance(minconf=1), 50) + + # Send 40 BTC from 0 to 1 and 60 BTC from 1 to 0. + txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')]) + self.nodes[0].sendrawtransaction(txs[0]['hex']) + self.nodes[1].sendrawtransaction(txs[0]['hex']) # sending on both nodes is faster than waiting for propagation + + self.sync_all() + txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), 60, [Decimal('0.01'), Decimal('0.02')]) + self.nodes[1].sendrawtransaction(txs[0]['hex']) + self.nodes[0].sendrawtransaction(txs[0]['hex']) # sending on both nodes is faster than waiting for propagation + self.sync_all() + + # First argument of getbalance must be set to "*" + assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "") + + self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + + # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions + assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('29.99')) # change from node 1's send + # Same with minconf=0 + assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99')) + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) + assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) + # getunconfirmedbalance + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + + # Node 1 bumps the transaction fee and resends + self.nodes[1].sendrawtransaction(txs[1]['hex']) + self.sync_all() + + self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) # output of node 1's send + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) + + self.nodes[1].generatetoaddress(1, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + # balances are correct after the transactions are confirmed + assert_equal(self.nodes[0].getbalance(), Decimal('69.99')) # node 1's send plus change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('29.98')) # change from node 0's send + + # Send total balance away from node 1 + txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) + self.nodes[1].sendrawtransaction(txs[0]['hex']) + self.nodes[1].generatetoaddress(2, RANDOM_COINBASE_ADDRESS) + self.sync_all() + + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + # getbalance with minconf=3 should still show the old balance + assert_equal(self.nodes[1].getbalance(minconf=3), Decimal('0')) + + # getbalance with minconf=2 will show the new balance. + assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0')) + + # check mempool transactions count for wallet unconfirmed balance after + # dynamically loading the wallet. + before = self.nodes[1].getunconfirmedbalance() + dst = self.nodes[1].getnewaddress() + self.nodes[1].unloadwallet('') + self.nodes[0].sendtoaddress(dst, 0.1) + self.sync_all() + self.nodes[1].loadwallet('') + after = self.nodes[1].getunconfirmedbalance() + assert_equal(before + Decimal('0.1'), after) + + +if __name__ == '__main__': + WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index c9b40905f0..daa834b5b8 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" @@ -11,14 +11,12 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_fee_amount, - assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, - sync_blocks, - sync_mempools, wait_until, ) + class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 @@ -34,7 +32,7 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size): """Return curr_balance after asserting the fee was in range""" @@ -59,23 +57,14 @@ class WalletTest(BitcoinTestFramework): assert_equal(walletinfo['immature_balance'], 50) assert_equal(walletinfo['balance'], 0) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) self.nodes[1].generate(101) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) assert_equal(self.nodes[2].getbalance(), 0) - # Check getbalance with different arguments - assert_equal(self.nodes[0].getbalance("*"), 50) - assert_equal(self.nodes[0].getbalance("*", 1), 50) - assert_equal(self.nodes[0].getbalance("*", 1, True), 50) - assert_equal(self.nodes[0].getbalance(minconf=1), 50) - - # first argument of getbalance must be excluded or set to "*" - assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[0].getbalance, "") - # Check that only first and second nodes have UTXOs utxos = self.nodes[0].listunspent() assert_equal(len(utxos), 1) @@ -92,13 +81,8 @@ class WalletTest(BitcoinTestFramework): assert_equal(txout['value'], 50) # Send 21 BTC from 0 to 2 using sendtoaddress call. - # Locked memory should increase to sign transactions - self.log.info("test getmemoryinfo") - memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - memory_after = self.nodes[0].getmemoryinfo() - assert_greater_than(memory_after['locked']['used'], memory_before['locked']['used']) self.log.info("test gettxout (second part)") # utxo spent in mempool should be visible if you exclude mempool @@ -122,7 +106,7 @@ class WalletTest(BitcoinTestFramework): # Have node0 mine a block, thus it will collect its own fee. self.nodes[0].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] @@ -158,7 +142,7 @@ class WalletTest(BitcoinTestFramework): # Have node1 generate 100 blocks (so node0 can recover the fee) self.nodes[1].generate(100) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # node0 should end up with 100 btc in block rewards plus fees, but # minus the 21 plus fees sent to node2 @@ -182,12 +166,12 @@ class WalletTest(BitcoinTestFramework): txns_to_send.append(self.nodes[0].signrawtransactionwithwallet(raw_tx)) # Have node 1 (miner) send the transactions - self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], True) - self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], True) + self.nodes[1].sendrawtransaction(txns_to_send[0]["hex"], 0) + self.nodes[1].sendrawtransaction(txns_to_send[1]["hex"], 0) # Have node1 mine a block to confirm transactions: self.nodes[1].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 94) @@ -202,55 +186,37 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].settxfee(fee_per_byte * 1000) txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + self.sync_all(self.nodes[0:3]) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount txid = self.nodes[2].sendtoaddress(address, 10, "", "", True) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) # Sendmany 10 BTC txid = self.nodes[2].sendmany('', {address: 10}, 0, "", []) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_0_bal += Decimal('10') - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), node_0_bal) # Sendmany 10 BTC with subtract fee from amount txid = self.nodes[2].sendmany('', {address: 10}, 0, "", [address]) self.nodes[2].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) - - # Test ResendWalletTransactions: - # Create a couple of transactions, then start up a fourth - # node (nodes[3]) and ask nodes[0] to rebroadcast. - # EXPECT: nodes[3] should have those transactions in its mempool. - txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - sync_mempools(self.nodes[0:2]) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) self.start_node(3) connect_nodes_bi(self.nodes, 0, 3) - sync_blocks(self.nodes) - - relayed = self.nodes[0].resendwallettransactions() - assert_equal(set(relayed), {txid1, txid2}) - sync_mempools(self.nodes) - - assert(txid1 in self.nodes[3].getrawmempool()) - - # Exercise balance rpcs - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], 1) - assert_equal(self.nodes[0].getunconfirmedbalance(), 1) + self.sync_all() # check if we can list zero value tx as available coins # 1. create raw_tx @@ -277,7 +243,7 @@ class WalletTest(BitcoinTestFramework): if uTx['txid'] == zero_value_txid: found = True assert_equal(uTx['amount'], Decimal('0')) - assert(found) + assert found # do some -walletbroadcast tests self.stop_nodes() @@ -287,18 +253,18 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) txid_not_broadcast = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) self.nodes[1].generate(1) # mine a block, tx should not be in there - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[2].getbalance(), node_2_bal) # should not be changed because tx was not broadcasted # now broadcast from another node, mine a block, sync, and check the balance self.nodes[1].sendrawtransaction(tx_obj_not_broadcast['hex']) self.nodes[1].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) node_2_bal += 2 tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) assert_equal(self.nodes[2].getbalance(), node_2_bal) @@ -314,10 +280,10 @@ class WalletTest(BitcoinTestFramework): connect_nodes_bi(self.nodes, 0, 1) connect_nodes_bi(self.nodes, 1, 2) connect_nodes_bi(self.nodes, 0, 2) - sync_blocks(self.nodes[0:3]) + self.sync_blocks(self.nodes[0:3]) self.nodes[0].generate(1) - sync_blocks(self.nodes[0:3]) + self.sync_blocks(self.nodes[0:3]) node_2_bal += 2 # tx should be added to balance because after restarting the nodes tx should be broadcast @@ -337,24 +303,50 @@ class WalletTest(BitcoinTestFramework): tx_obj = self.nodes[0].gettransaction(txid) assert_equal(tx_obj['amount'], Decimal('-0.0001')) + # General checks for errors from incorrect inputs # This will raise an exception because the amount type is wrong assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") + # This will raise an exception for the invalid private key format + assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") + + # This will raise an exception for importing an address with the PS2H flag + temp_address = self.nodes[1].getnewaddress() + assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) + + # This will raise an exception for attempting to dump the private key of an address you do not own + assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) + + # This will raise an exception for attempting to get the private key of an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") + + # This will raise an exception for attempting to set a label for an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") + + # This will raise an exception for importing an invalid address + assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") + + # This will raise an exception for attempting to import a pubkey that isn't in hex + assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") + + # This will raise an exception for importing an invalid pubkey + assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + # Import address and private key to check correct behavior of spendable unspents # 1. Send some coins to generate new UTXO address_to_import = self.nodes[2].getnewaddress() txid = self.nodes[0].sendtoaddress(address_to_import, 1) self.nodes[0].generate(1) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # 2. Import address from node2 to node1 self.nodes[1].importaddress(address_to_import) # 3. Validate that the imported address is watch-only on node1 - assert(self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"]) + assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] # 4. Check that the unspents after import are not spendable assert_array_result(self.nodes[1].listunspent(), @@ -374,15 +366,15 @@ class WalletTest(BitcoinTestFramework): coinbase_addr = self.nodes[1].getnewaddress() block_hash = self.nodes[0].generatetoaddress(1, coinbase_addr)[0] coinbase_txid = self.nodes[0].getblock(block_hash)['tx'][0] - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) # Check that the txid and balance is found by node1 self.nodes[1].gettransaction(coinbase_txid) # check if wallet or blockchain maintenance changes the balance - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) blocks = self.nodes[0].generate(2) - self.sync_all([self.nodes[0:3]]) + self.sync_all(self.nodes[0:3]) balance_nodes = [self.nodes[i].getbalance() for i in range(3)] block_count = self.nodes[0].getblockcount() @@ -396,7 +388,7 @@ class WalletTest(BitcoinTestFramework): addr = self.nodes[0].getnewaddress() self.nodes[0].setlabel(addr, label) assert_equal(self.nodes[0].getaddressinfo(addr)['label'], label) - assert(label in self.nodes[0].listlabels()) + assert label in self.nodes[0].listlabels() self.nodes[0].rpc.ensure_ascii = True # restore to default # maintenance tests @@ -455,8 +447,8 @@ class WalletTest(BitcoinTestFramework): # Without walletrejectlongchains, we will still generate a txid # The tx will be stored in the wallet but not accepted to the mempool extra_txid = self.nodes[0].sendtoaddress(sending_addr, Decimal('0.0001')) - assert(extra_txid not in self.nodes[0].getrawmempool()) - assert(extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()]) + assert extra_txid not in self.nodes[0].getrawmempool() + assert extra_txid in [tx["txid"] for tx in self.nodes[0].listtransactions()] self.nodes[0].abandontransaction(extra_txid) total_txs = len(self.nodes[0].listtransactions("*", 99999)) @@ -493,7 +485,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].generate(1) destination = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(destination, 0.123) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].getrawtransaction(txid)) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) output_addresses = [vout['scriptPubKey']['addresses'][0] for vout in tx["vout"]] assert len(output_addresses) > 1 for address in output_addresses: @@ -504,5 +496,6 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].setlabel(change, 'foobar') assert_equal(self.nodes[0].getaddressinfo(change)['ischange'], False) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 7d3d9b61e2..568b1f28d8 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -19,7 +19,13 @@ import io from test_framework.blocktools import add_witness_commitment, create_block, create_coinbase, send_to_witness from test_framework.messages import BIP125_SEQUENCE_NUMBER, CTransaction from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes, sync_mempools +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + connect_nodes_bi, + hex_str_to_bytes, +) WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -60,7 +66,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address) + test_simple_bumpfee_succeeds(self, rbf_node, peer_node, dest_address) test_segwit_bumpfee_succeeds(rbf_node, dest_address) test_nonrbf_bumpfee_fails(peer_node, dest_address) test_notmine_bumpfee_fails(rbf_node, peer_node, dest_address) @@ -73,19 +79,23 @@ class BumpFeeTest(BitcoinTestFramework): test_unconfirmed_not_spendable(rbf_node, rbf_node_address) test_bumpfee_metadata(rbf_node, dest_address) test_locked_wallet_fails(rbf_node, dest_address) + test_change_script_match(rbf_node, dest_address) + # These tests wipe out a number of utxos that are expected in other tests + test_small_output_with_feerate_succeeds(rbf_node, dest_address) + test_no_more_inputs_fails(rbf_node, dest_address) self.log.info("Success") -def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): +def test_simple_bumpfee_succeeds(self, rbf_node, peer_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) rbftx = rbf_node.gettransaction(rbfid) - sync_mempools((rbf_node, peer_node)) + self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 # check that bumped_tx propagates, original tx was evicted and has a wallet conflict - sync_mempools((rbf_node, peer_node)) + self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert bumped_tx["txid"] in peer_node.getrawmempool() assert rbfid not in rbf_node.getrawmempool() @@ -173,6 +183,40 @@ def test_small_output_fails(rbf_node, dest_address): rbfid = spend_one_input(rbf_node, dest_address) assert_raises_rpc_error(-4, "Change output is too small", rbf_node.bumpfee, rbfid, {"totalFee": 50001}) +def test_small_output_with_feerate_succeeds(rbf_node, dest_address): + + # Make sure additional inputs exist + rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) + rbfid = spend_one_input(rbf_node, dest_address) + original_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_equal(len(original_input_list), 1) + original_txin = original_input_list[0] + # Keep bumping until we out-spend change output + tx_fee = 0 + while tx_fee < Decimal("0.0005"): + new_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + new_item = list(new_input_list)[0] + assert_equal(len(original_input_list), 1) + assert_equal(original_txin["txid"], new_item["txid"]) + assert_equal(original_txin["vout"], new_item["vout"]) + rbfid_new_details = rbf_node.bumpfee(rbfid) + rbfid_new = rbfid_new_details["txid"] + raw_pool = rbf_node.getrawmempool() + assert rbfid not in raw_pool + assert rbfid_new in raw_pool + rbfid = rbfid_new + tx_fee = rbfid_new_details["origfee"] + + # input(s) have been added + final_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_greater_than(len(final_input_list), 1) + # Original input is in final set + assert [txin for txin in final_input_list + if txin["txid"] == original_txin["txid"] + and txin["vout"] == original_txin["vout"]] + + rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) + assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) def test_dust_to_fee(rbf_node, dest_address): # check that if output is reduced to dust, it will be converted to fee @@ -272,19 +316,37 @@ def test_locked_wallet_fails(rbf_node, dest_address): rbf_node.walletlock() assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + +def test_change_script_match(rbf_node, dest_address): + """Test that the same change addresses is used for the replacement transaction when possible.""" + def get_change_address(tx): + tx_details = rbf_node.getrawtransaction(tx, 1) + txout_addresses = [txout['scriptPubKey']['addresses'][0] for txout in tx_details["vout"]] + return [address for address in txout_addresses if rbf_node.getaddressinfo(address)["ischange"]] + # Check that there is only one change output + rbfid = spend_one_input(rbf_node, dest_address) + change_addresses = get_change_address(rbfid) + assert_equal(len(change_addresses), 1) + + # Now find that address in each subsequent tx, and no other change + bumped_total_tx = rbf_node.bumpfee(rbfid, {"totalFee": 2000}) + assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'])) + bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) + assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) -def spend_one_input(node, dest_address): +def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): tx_input = dict( sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) - rawtx = node.createrawtransaction( - [tx_input], {dest_address: Decimal("0.00050000"), - node.getrawchangeaddress(): Decimal("0.00049000")}) + destinations = {dest_address: Decimal("0.00050000")} + if change_size > 0: + destinations[node.getrawchangeaddress()] = change_size + rawtx = node.createrawtransaction([tx_input], destinations) signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx["hex"]) return txid - def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -298,9 +360,15 @@ def submit_block_with_tx(node, tx): block.hashMerkleRoot = block.calc_merkle_root() add_witness_commitment(block) block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) return block +def test_no_more_inputs_fails(rbf_node, dest_address): + # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient + rbf_node.generatetoaddress(1, dest_address) + # spend all funds, no change output + rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) + assert_raises_rpc_error(-4, "Unable to create transaction: Insufficient funds", rbf_node.bumpfee, rbfid) if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_coinbase_category.py b/test/functional/wallet_coinbase_category.py new file mode 100755 index 0000000000..7aa8b44ebd --- /dev/null +++ b/test/functional/wallet_coinbase_category.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test coinbase transactions return the correct categories. + +Tests listtransactions, listsinceblock, and gettransaction. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_array_result +) + +class CoinbaseCategoryTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def assert_category(self, category, address, txid, skip): + assert_array_result(self.nodes[0].listtransactions(skip=skip), + {"address": address}, + {"category": category}) + assert_array_result(self.nodes[0].listsinceblock()["transactions"], + {"address": address}, + {"category": category}) + assert_array_result(self.nodes[0].gettransaction(txid)["details"], + {"address": address}, + {"category": category}) + + def run_test(self): + # Generate one block to an address + address = self.nodes[0].getnewaddress() + self.nodes[0].generatetoaddress(1, address) + hash = self.nodes[0].getbestblockhash() + txid = self.nodes[0].getblock(hash)["tx"][0] + + # Coinbase transaction is immature after 1 confirmation + self.assert_category("immature", address, txid, 0) + + # Mine another 99 blocks on top + self.nodes[0].generate(99) + # Coinbase transaction is still immature after 100 confirmations + self.assert_category("immature", address, txid, 99) + + # Mine one more block + self.nodes[0].generate(1) + # Coinbase transaction is now matured, so category is "generate" + self.assert_category("generate", address, txid, 100) + + # Orphan block that paid to address + self.nodes[0].invalidateblock(hash) + # Coinbase transaction is now orphaned + self.assert_category("orphan", address, txid, 100) + +if __name__ == '__main__': + CoinbaseCategoryTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py new file mode 100755 index 0000000000..0b584a0bb2 --- /dev/null +++ b/test/functional/wallet_create_tx.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) +from test_framework.blocktools import ( + TIME_GENESIS_BLOCK, +) + + +class CreateTxWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.log.info('Create some old blocks') + self.nodes[0].setmocktime(TIME_GENESIS_BLOCK) + self.nodes[0].generate(200) + self.nodes[0].setmocktime(0) + + self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + assert_equal(tx['locktime'], 0) + + self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block') + self.nodes[0].generate(1) + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + assert 0 < tx['locktime'] <= 201 + + +if __name__ == '__main__': + CreateTxWalletTest().main() diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py new file mode 100755 index 0000000000..8ec4b98b6e --- /dev/null +++ b/test/functional/wallet_createwallet.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test createwallet arguments. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +class CreateWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + node.generate(1) # Leave IBD for sethdseed + + self.nodes[0].createwallet(wallet_name='w0') + w0 = node.get_wallet_rpc('w0') + address1 = w0.getnewaddress() + + self.log.info("Test disableprivatekeys creation.") + self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True) + w1 = node.get_wallet_rpc('w1') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getrawchangeaddress) + w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info('Test that private keys cannot be imported') + addr = w0.getnewaddress('', 'legacy') + privkey = w0.dumpprivkey(addr) + assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) + result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) + assert not result[0]['success'] + assert 'warning' not in result[0] + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Test blank creation with private keys disabled.") + self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True) + w2 = node.get_wallet_rpc('w2') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getrawchangeaddress) + w2.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info("Test blank creation with private keys enabled.") + self.nodes[0].createwallet(wallet_name='w3', disable_private_keys=False, blank=True) + w3 = node.get_wallet_rpc('w3') + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress) + # Import private key + w3.importprivkey(w0.dumpprivkey(address1)) + # Imported private keys are currently ignored by the keypool + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + # Set the seed + w3.sethdseed() + assert_equal(w3.getwalletinfo()['keypoolsize'], 1) + w3.getnewaddress() + w3.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys enabled and then encryption") + self.nodes[0].createwallet(wallet_name='w4', disable_private_keys=False, blank=True) + w4 = node.get_wallet_rpc('w4') + assert_equal(w4.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Encrypt the wallet. Nothing should change about the keypool + w4.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Now set a seed and it should work. Wallet should also be encrypted + w4.walletpassphrase('pass', 2) + w4.sethdseed() + w4.getnewaddress() + w4.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys disabled and then encryption") + self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True) + w5 = node.get_wallet_rpc('w5') + assert_equal(w5.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) + # Encrypt the wallet + w5.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) + +if __name__ == '__main__': + CreateWalletTest().main() diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 6530c58c78..7c2ec56b5a 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. @@ -21,9 +21,9 @@ class DisableWalletTest (BitcoinTestFramework): # Make sure wallet is really disabled assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') - assert(x['isvalid'] == False) + assert x['isvalid'] == False x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert(x['isvalid'] == True) + assert x['isvalid'] == True # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. diff --git a/test/functional/wallet_disableprivatekeys.py b/test/functional/wallet_disableprivatekeys.py deleted file mode 100755 index 34ff525255..0000000000 --- a/test/functional/wallet_disableprivatekeys.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test disable-privatekeys mode. -""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_raises_rpc_error, -) - - -class DisablePrivateKeysTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = False - self.num_nodes = 1 - self.supports_cli = True - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - node = self.nodes[0] - self.log.info("Test disableprivatekeys creation.") - self.nodes[0].createwallet('w1', True) - self.nodes[0].createwallet('w2') - w1 = node.get_wallet_rpc('w1') - w2 = node.get_wallet_rpc('w2') - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getnewaddress) - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) - w1.importpubkey(w2.getaddressinfo(w2.getnewaddress())['pubkey']) - -if __name__ == '__main__': - DisablePrivateKeysTest().main() diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 20cb816ee8..53edf710b9 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" @@ -46,10 +46,10 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): keypath = None if keytype == "inactivehdseed=1": # ensure the old master is still available - assert (hd_master_addr_old == addr) + assert hd_master_addr_old == addr elif keytype == "hdseed=1": # ensure we have generated a new hd master key - assert (hd_master_addr_old != addr) + assert hd_master_addr_old != addr hd_master_addr_ret = addr elif keytype == "script=1": # scripts don't have keypaths @@ -94,7 +94,7 @@ class WalletDumpTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self, split=False): + def setup_network(self): self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index ab9ebed8d4..c514b7e0b4 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -31,12 +31,18 @@ class WalletEncryptionTest(BitcoinTestFramework): privkey = self.nodes[0].dumpprivkey(address) assert_equal(privkey[:1], "c") assert_equal(len(privkey), 52) + assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrase was called", self.nodes[0].walletpassphrase, 'ff', 1) + assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff') # Encrypt the wallet + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].encryptwallet, '') self.nodes[0].encryptwallet(passphrase) # Test that the wallet is encrypted assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff') + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1) + assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff') # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 9d61483868..5452433acf 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -21,7 +21,7 @@ class WalletGroupTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] - self.rpc_timewait = 120 + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index eb42531693..97172d8b82 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" @@ -27,7 +27,6 @@ class WalletHDTest(BitcoinTestFramework): def run_test(self): # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert_equal(masterkeyid, self.nodes[1].getwalletinfo()['hdmasterkeyid']) assert_equal(len(masterkeyid), 40) # create an internal key @@ -53,7 +52,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info = self.nodes[1].getaddressinfo(hd_add) assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info["hdseedid"], masterkeyid) - assert_equal(hd_info["hdmasterkeyid"], masterkeyid) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) @@ -83,7 +81,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info_2["hdseedid"], masterkeyid) - assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid) assert_equal(hd_add, hd_add_2) connect_nodes_bi(self.nodes, 0, 1) self.sync_all() diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 46462a16f3..9de30d0374 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet import RPCs. @@ -20,7 +20,12 @@ happened previously. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_raises_rpc_error, connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import ( + assert_raises_rpc_error, + connect_nodes, + assert_equal, + set_node_times, +) import collections import enum @@ -161,11 +166,12 @@ class ImportRescanTest(BitcoinTestFramework): timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] set_node_times(self.nodes, timestamp + TIMESTAMP_WINDOW + 1) self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_all() # For each variation of wallet key import, invoke the import RPC and # check the results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run import for variant {}'.format(variant)) variant.expect_disabled = variant.rescan == Rescan.yes and variant.prune and variant.call == Call.single expect_rescan = variant.rescan == Rescan.yes and not variant.expect_disabled variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] @@ -187,10 +193,11 @@ class ImportRescanTest(BitcoinTestFramework): # Generate a block containing the new transactions. self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) - sync_blocks(self.nodes) + self.sync_all() # Check the latest results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run check for variant {}'.format(variant)) if not variant.expect_disabled: variant.expected_balance += variant.sent_amount variant.expected_txs += 1 diff --git a/test/functional/wallet_import_with_label.py b/test/functional/wallet_import_with_label.py index 95acaa752e..a623b75606 100755 --- a/test/functional/wallet_import_with_label.py +++ b/test/functional/wallet_import_with_label.py @@ -11,7 +11,7 @@ with and without a label. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.wallet_util import test_address class ImportWithLabel(BitcoinTestFramework): @@ -32,11 +32,11 @@ class ImportWithLabel(BitcoinTestFramework): address = self.nodes[0].getnewaddress() label = "Test Label" self.nodes[1].importaddress(address, label) - address_assert = self.nodes[1].getaddressinfo(address) - - assert_equal(address_assert["iswatchonly"], True) - assert_equal(address_assert["ismine"], False) - assert_equal(address_assert["label"], label) + test_address(self.nodes[1], + address, + iswatchonly=True, + ismine=False, + label=label) self.log.info( "Import the watch-only address's private key without a " @@ -45,7 +45,9 @@ class ImportWithLabel(BitcoinTestFramework): priv_key = self.nodes[0].dumpprivkey(address) self.nodes[1].importprivkey(priv_key) - assert_equal(label, self.nodes[1].getaddressinfo(address)["label"]) + test_address(self.nodes[1], + address, + label=label) self.log.info( "Test importaddress without label and importprivkey with label." @@ -53,11 +55,11 @@ class ImportWithLabel(BitcoinTestFramework): self.log.info("Import a watch-only address without a label.") address2 = self.nodes[0].getnewaddress() self.nodes[1].importaddress(address2) - address_assert2 = self.nodes[1].getaddressinfo(address2) - - assert_equal(address_assert2["iswatchonly"], True) - assert_equal(address_assert2["ismine"], False) - assert_equal(address_assert2["label"], "") + test_address(self.nodes[1], + address2, + iswatchonly=True, + ismine=False, + label="") self.log.info( "Import the watch-only address's private key with a " @@ -67,18 +69,20 @@ class ImportWithLabel(BitcoinTestFramework): label2 = "Test Label 2" self.nodes[1].importprivkey(priv_key2, label2) - assert_equal(label2, self.nodes[1].getaddressinfo(address2)["label"]) + test_address(self.nodes[1], + address2, + label=label2) self.log.info("Test importaddress with label and importprivkey with label.") self.log.info("Import a watch-only address with a label.") address3 = self.nodes[0].getnewaddress() label3_addr = "Test Label 3 for importaddress" self.nodes[1].importaddress(address3, label3_addr) - address_assert3 = self.nodes[1].getaddressinfo(address3) - - assert_equal(address_assert3["iswatchonly"], True) - assert_equal(address_assert3["ismine"], False) - assert_equal(address_assert3["label"], label3_addr) + test_address(self.nodes[1], + address3, + iswatchonly=True, + ismine=False, + label=label3_addr) self.log.info( "Import the watch-only address's private key with a " @@ -88,7 +92,9 @@ class ImportWithLabel(BitcoinTestFramework): label3_priv = "Test Label 3 for importprivkey" self.nodes[1].importprivkey(priv_key3, label3_priv) - assert_equal(label3_priv, self.nodes[1].getaddressinfo(address3)["label"]) + test_address(self.nodes[1], + address3, + label=label3_priv) self.log.info( "Test importprivkey won't label new dests with the same " @@ -98,15 +104,12 @@ class ImportWithLabel(BitcoinTestFramework): address4 = self.nodes[0].getnewaddress() label4_addr = "Test Label 4 for importaddress" self.nodes[1].importaddress(address4, label4_addr) - address_assert4 = self.nodes[1].getaddressinfo(address4) - - assert_equal(address_assert4["iswatchonly"], True) - assert_equal(address_assert4["ismine"], False) - assert_equal(address_assert4["label"], label4_addr) - - self.log.info("Asserts address has no embedded field with dests.") - - assert_equal(address_assert4.get("embedded"), None) + test_address(self.nodes[1], + address4, + iswatchonly=True, + ismine=False, + label=label4_addr, + embedded=None) self.log.info( "Import the watch-only address's private key without a " @@ -116,16 +119,14 @@ class ImportWithLabel(BitcoinTestFramework): ) priv_key4 = self.nodes[0].dumpprivkey(address4) self.nodes[1].importprivkey(priv_key4) - address_assert4 = self.nodes[1].getaddressinfo(address4) - - assert address_assert4.get("embedded") - - bcaddress_assert = self.nodes[1].getaddressinfo( - address_assert4["embedded"]["address"] - ) - - assert_equal(address_assert4["label"], label4_addr) - assert_equal(bcaddress_assert["label"], "") + embedded_addr = self.nodes[1].getaddressinfo(address4)['embedded']['address'] + + test_address(self.nodes[1], + embedded_addr, + label="") + test_address(self.nodes[1], + address4, + label=label4_addr) self.stop_nodes() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 5c789b1c03..81c650f4c1 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,24 +1,36 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the importmulti RPC.""" +"""Test the importmulti RPC. -from test_framework import script +Test importmulti by generating keys on node0, importing the scriptPubKeys and +addresses on node1 and then testing the address info for the different address +variants. + +- `get_key()` and `get_multisig()` are called to generate keys on node0 and + return the privkeys, pubkeys and all variants of scriptPubKey and address. +- `test_importmulti()` is called to send an importmulti call to node1, test + success, and (if unsuccessful) test the error code and error message returned. +- `test_address()` is called to call getaddressinfo for an address on node1 + and test the values returned.""" + +from test_framework.script import ( + CScript, + OP_NOP, +) from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - bytes_to_hex_str, - hex_str_to_bytes ) -from test_framework.script import ( - CScript, - OP_0, - hash160 +from test_framework.wallet_util import ( + get_key, + get_multisig, + test_address, ) -from test_framework.messages import sha256 class ImportMultiTest(BitcoinTestFramework): def set_test_params(self): @@ -32,7 +44,19 @@ class ImportMultiTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - def run_test (self): + def test_importmulti(self, req, success, error_code=None, error_message=None, warnings=[]): + """Run importmulti and assert success""" + result = self.nodes[1].importmulti([req]) + observed_warnings = [] + if 'warnings' in result[0]: + observed_warnings = result[0]['warnings'] + assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) + assert_equal(result[0]['success'], success) + if error_code is not None: + assert_equal(result[0]['error']['code'], error_code) + assert_equal(result[0]['error']['message'], error_message) + + def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(1) self.nodes[1].generate(1) @@ -40,587 +64,755 @@ class ImportMultiTest(BitcoinTestFramework): node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - #Check only one address + # Check only one address assert_equal(node0_address1['ismine'], True) - #Node 1 sync test - assert_equal(self.nodes[1].getblockcount(),1) + # Node 1 sync test + assert_equal(self.nodes[1].getblockcount(), 1) - #Address Test - before import + # Address Test - before import address_info = self.nodes[1].getaddressinfo(node0_address1['address']) assert_equal(address_info['iswatchonly'], False) assert_equal(address_info['ismine'], False) - # RPC importmulti ----------------------------------------------- # Bitcoin Address (implicit non-internal) self.log.info("Should import an address") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) - assert_equal(address_assert['ischange'], False) - watchonly_address = address['address'] + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp, + ischange=False) + watchonly_address = key.p2pkh_addr watchonly_timestamp = timestamp self.log.info("Should not import an invalid address") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": "not valid address", - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Invalid address') + self.test_importmulti({"scriptPubKey": {"address": "not valid address"}, + "timestamp": "now"}, + success=False, + error_code=-5, + error_message='Invalid address \"not valid address\"') # ScriptPubKey + internal self.log.info("Should import a scriptPubKey with internal flag") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "internal": True - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) - assert_equal(address_assert['ischange'], True) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "internal": True}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp, + ischange=True) # ScriptPubKey + internal + label self.log.info("Should not allow a label to be specified when internal is true") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "internal": True, - "label": "Example label" - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal addresses should not have a label') + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "internal": True, + "label": "Example label"}, + success=False, + error_code=-8, + error_message='Internal addresses should not have a label') # Nonstandard scriptPubKey + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal flag") - nonstandardScriptPubKey = address['scriptPubKey'] + bytes_to_hex_str(script.CScript([script.OP_NOP])) - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + nonstandardScriptPubKey = key.p2pkh_script + CScript([OP_NOP]).hex() + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) # Address + Public key + !Internal(explicit) self.log.info("Should import an address with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "pubkeys": [ address['pubkey'] ], - "internal": False - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) - + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "pubkeys": [key.pubkey], + "internal": False}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp) # ScriptPubKey + Public key + internal self.log.info("Should import a scriptPubKey with internal and with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "pubkeys": [ address['pubkey'] ], - "internal": True - }] - result = self.nodes[1].importmulti(requests=request) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "pubkeys": [key.pubkey], + "internal": True}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + timestamp=timestamp) # Nonstandard scriptPubKey + Public key + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal and with public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - "pubkeys": [ address['pubkey'] ] - }] - result = self.nodes[1].importmulti(requests=request) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now", + "pubkeys": [key.pubkey]}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) # Address + Private key + !watchonly self.log.info("Should import an address with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], True) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) self.log.info("Should not import an address with private key if is already imported") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -4) - assert_equal(result[0]['error']['message'], 'The wallet already contains the private key for this address or script') + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=False, + error_code=-4, + error_message='The wallet already contains the private key for this address or script ("' + key.p2pkh_script + '")') # Address + Private key + watchonly - self.log.info("Should not import an address with private key and with watchonly") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ], - "watchonly": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Watch-only addresses should not include private keys') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) + self.log.info("Should import an address with private key and with watchonly") + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [key.privkey], + "watchonly": True}, + success=True, + warnings=["All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) # ScriptPubKey + Private key + internal self.log.info("Should import a scriptPubKey with internal and with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ], - "internal": True - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], True) - assert_equal(address_assert['timestamp'], timestamp) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "keys": [key.privkey], + "internal": True}, + success=True) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=True, + timestamp=timestamp) # Nonstandard scriptPubKey + Private key + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal and with private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": nonstandardScriptPubKey, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Internal must be set to true for nonstandard scriptPubKey imports.') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, + "timestamp": "now", + "keys": [key.privkey]}, + success=False, + error_code=-8, + error_message='Internal must be set to true for nonstandard scriptPubKey imports.') + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=False, + ismine=False, + timestamp=None) # P2SH address - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['isscript'], True) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['timestamp'], timestamp) - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + isscript=True, + iswatchonly=True, + timestamp=timestamp) + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], False) - # P2SH + Redeem script - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['timestamp'], timestamp) - - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_addr, timestamp=timestamp, iswatchonly=True, ismine=False, solvable=True) + + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], True) - # P2SH + Redeem script + Private Keys + !Watchonly - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'], - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['timestamp'], timestamp) - - p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script, + "keys": multisig.privkeys[0:2]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_addr, + timestamp=timestamp, + ismine=False, + iswatchonly=True, + solvable=True) + + p2shunspent = self.nodes[1].listunspent(0, 999999, [multisig.p2sh_addr])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], True) # P2SH + Redeem script + Private Keys + Watchonly - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_3 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].createmultisig(2, [sig_address_1['pubkey'], sig_address_2['pubkey'], sig_address_3['pubkey']]) + multisig = get_multisig(self.nodes[0]) self.nodes[1].generate(100) - self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) + self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.log.info("Should import a p2sh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "redeemscript": multi_sig_script['redeemScript'], - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])], - "watchonly": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -8) - assert_equal(result[0]['error']['message'], 'Watch-only addresses should not include private keys') - + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, + "timestamp": "now", + "redeemscript": multisig.redeem_script, + "keys": multisig.privkeys[0:2], + "watchonly": True}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + iswatchonly=True, + ismine=False, + solvable=True, + timestamp=timestamp) # Address + Public key + !Internal + Wrong pubkey - self.log.info("Should not import an address with a wrong public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "pubkeys": [ address2['pubkey'] ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Key does not match address destination') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should not import an address with the wrong public key as non-solvable") + key = get_key(self.nodes[0]) + wrong_key = get_key(self.nodes[0]).pubkey + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "pubkeys": [wrong_key]}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # ScriptPubKey + Public key + internal + Wrong pubkey - self.log.info("Should not import a scriptPubKey with internal and with a wrong public key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - request = [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "pubkeys": [ address2['pubkey'] ], - "internal": True - }] - result = self.nodes[1].importmulti(request) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Key does not match address destination') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import a scriptPubKey with internal and with a wrong public key as non-solvable") + key = get_key(self.nodes[0]) + wrong_key = get_key(self.nodes[0]).pubkey + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "pubkeys": [wrong_key], + "internal": True}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # Address + Private key + !watchonly + Wrong private key - self.log.info("Should not import an address with a wrong private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address2['address']) ] - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Key does not match address destination') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import an address with a wrong private key as non-solvable") + key = get_key(self.nodes[0]) + wrong_privkey = get_key(self.nodes[0]).privkey + self.test_importmulti({"scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now", + "keys": [wrong_privkey]}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # ScriptPubKey + Private key + internal + Wrong private key - self.log.info("Should not import a scriptPubKey with internal and with a wrong private key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - address2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "now", - "keys": [ self.nodes[0].dumpprivkey(address2['address']) ], - "internal": True - }]) - assert_equal(result[0]['success'], False) - assert_equal(result[0]['error']['code'], -5) - assert_equal(result[0]['error']['message'], 'Key does not match address destination') - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], False) - assert_equal('timestamp' in address_assert, False) - + self.log.info("Should import a scriptPubKey with internal and with a wrong private key as non-solvable") + key = get_key(self.nodes[0]) + wrong_privkey = get_key(self.nodes[0]).privkey + self.test_importmulti({"scriptPubKey": key.p2pkh_script, + "timestamp": "now", + "keys": [wrong_privkey], + "internal": True}, + success=True, + warnings=["Importing as non-solvable: some required keys are missing. If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript.", "Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + iswatchonly=True, + ismine=False, + solvable=False, + timestamp=timestamp) # Importing existing watch only address with new timestamp should replace saved timestamp. assert_greater_than(timestamp, watchonly_timestamp) self.log.info("Should replace previously saved watch only timestamp.") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": watchonly_address, - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(watchonly_address) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], timestamp) + self.test_importmulti({"scriptPubKey": {"address": watchonly_address}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + watchonly_address, + iswatchonly=True, + ismine=False, + timestamp=timestamp) watchonly_timestamp = timestamp - # restart nodes to check for proper serialization/deserialization of watch only address self.stop_nodes() self.start_nodes() - address_assert = self.nodes[1].getaddressinfo(watchonly_address) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['timestamp'], watchonly_timestamp) + test_address(self.nodes[1], + watchonly_address, + iswatchonly=True, + ismine=False, + timestamp=watchonly_timestamp) # Bad or missing timestamps self.log.info("Should throw on invalid or missing timestamp values") assert_raises_rpc_error(-3, 'Missing required timestamp field for key', - self.nodes[1].importmulti, [{ - "scriptPubKey": address['scriptPubKey'], - }]) + self.nodes[1].importmulti, [{"scriptPubKey": key.p2pkh_script}]) assert_raises_rpc_error(-3, 'Expected number or "now" timestamp value for key. got type string', - self.nodes[1].importmulti, [{ - "scriptPubKey": address['scriptPubKey'], - "timestamp": "", - }]) + self.nodes[1].importmulti, [{ + "scriptPubKey": key.p2pkh_script, + "timestamp": "" + }]) # Import P2WPKH address as watch only self.log.info("Should import a P2WPKH address as watch only") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress(address_type="bech32")) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], True) - assert_equal(address_assert['solvable'], False) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2wpkh_addr, + iswatchonly=True, + solvable=False) # Import P2WPKH address with public key but no private key self.log.info("Should import a P2WPKH address and public key as solvable but not spendable") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress(address_type="bech32")) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "pubkeys": [ address['pubkey'] ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['ismine'], False) - assert_equal(address_assert['solvable'], True) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now", + "pubkeys": [key.pubkey]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2wpkh_addr, + ismine=False, + solvable=True) # Import P2WPKH address with key and check it is spendable self.log.info("Should import a P2WPKH address with key") - address = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress(address_type="bech32")) - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": address['address'] - }, - "timestamp": "now", - "keys": [self.nodes[0].dumpprivkey(address['address'])] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(address['address']) - assert_equal(address_assert['iswatchonly'], False) - assert_equal(address_assert['ismine'], True) + key = get_key(self.nodes[0]) + self.test_importmulti({"scriptPubKey": {"address": key.p2wpkh_addr}, + "timestamp": "now", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2wpkh_addr, + iswatchonly=False, + ismine=True) # P2WSH multisig address without scripts or keys - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - sig_address_2 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].addmultisigaddress(2, [sig_address_1['pubkey'], sig_address_2['pubkey']], "", "bech32") + multisig = get_multisig(self.nodes[0]) self.log.info("Should import a p2wsh multisig as watch only without respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now" - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['solvable'], False) + self.test_importmulti({"scriptPubKey": {"address": multisig.p2wsh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + solvable=False) # Same P2WSH multisig address as above, but now with witnessscript + private keys - self.log.info("Should import a p2wsh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] - }, - "timestamp": "now", - "witnessscript": multi_sig_script['redeemScript'], - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address']) ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['solvable'], True) - assert_equal(address_assert['ismine'], True) - assert_equal(address_assert['sigsrequired'], 2) + self.log.info("Should import a p2wsh with respective witness script and private keys") + self.test_importmulti({"scriptPubKey": {"address": multisig.p2wsh_addr}, + "timestamp": "now", + "witnessscript": multisig.redeem_script, + "keys": multisig.privkeys}, + success=True) + test_address(self.nodes[1], + multisig.p2sh_addr, + solvable=True, + ismine=True, + sigsrequired=2) # P2SH-P2WPKH address with no redeemscript or public or private key - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress(address_type="p2sh-segwit")) - pubkeyhash = hash160(hex_str_to_bytes(sig_address_1['pubkey'])) - pkscript = CScript([OP_0, pubkeyhash]) + key = get_key(self.nodes[0]) self.log.info("Should import a p2sh-p2wpkh without redeem script or keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": sig_address_1['address'] - }, - "timestamp": "now" - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(sig_address_1['address']) - assert_equal(address_assert['solvable'], False) - assert_equal(address_assert['ismine'], False) + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now"}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=False, + ismine=False) # P2SH-P2WPKH address + redeemscript + public key with no private key self.log.info("Should import a p2sh-p2wpkh with respective redeem script and pubkey as solvable") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": sig_address_1['address'] - }, - "timestamp": "now", - "redeemscript": bytes_to_hex_str(pkscript), - "pubkeys": [ sig_address_1['pubkey'] ] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(sig_address_1['address']) - assert_equal(address_assert['solvable'], True) - assert_equal(address_assert['ismine'], False) + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now", + "redeemscript": key.p2sh_p2wpkh_redeem_script, + "pubkeys": [key.pubkey]}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=False) # P2SH-P2WPKH address + redeemscript + private key - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress(address_type="p2sh-segwit")) - pubkeyhash = hash160(hex_str_to_bytes(sig_address_1['pubkey'])) - pkscript = CScript([OP_0, pubkeyhash]) + key = get_key(self.nodes[0]) self.log.info("Should import a p2sh-p2wpkh with respective redeem script and private keys") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": sig_address_1['address'] - }, - "timestamp": "now", - "redeemscript": bytes_to_hex_str(pkscript), - "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address'])] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(sig_address_1['address']) - assert_equal(address_assert['solvable'], True) - assert_equal(address_assert['ismine'], True) - - # P2SH-P2WSH 1-of-1 multisig + redeemscript with no private key - sig_address_1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) - multi_sig_script = self.nodes[0].addmultisigaddress(1, [sig_address_1['pubkey']], "", "p2sh-segwit") - scripthash = sha256(hex_str_to_bytes(multi_sig_script['redeemScript'])) - redeem_script = CScript([OP_0, scripthash]) + self.test_importmulti({"scriptPubKey": {"address": key.p2sh_p2wpkh_addr}, + "timestamp": "now", + "redeemscript": key.p2sh_p2wpkh_redeem_script, + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True) + + # P2SH-P2WSH multisig + redeemscript with no private key + multisig = get_multisig(self.nodes[0]) self.log.info("Should import a p2sh-p2wsh with respective redeem script but no private key") - result = self.nodes[1].importmulti([{ - "scriptPubKey": { - "address": multi_sig_script['address'] + self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_p2wsh_addr}, + "timestamp": "now", + "redeemscript": multisig.p2wsh_script, + "witnessscript": multisig.redeem_script}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + multisig.p2sh_p2wsh_addr, + solvable=True, + ismine=False) + + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should not import a p2sh-p2wpkh address from descriptor without checksum and private key") + self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=False, + error_code=-5, + error_message="Descriptor is invalid") + + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.test_importmulti({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True, + label="Descriptor import test") + + # Test ranged descriptor fails if range is not specified + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.log.info("Ranged descriptor import should fail without a specified range") + self.test_importmulti({"desc": descsum_create(desc), + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor is ranged, please specify the range') + + # Test importing of a ranged descriptor without keys + self.log.info("Should import the ranged descriptor with specified range as solvable") + self.test_importmulti({"desc": descsum_create(desc), + "timestamp": "now", + "range": 1}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + for address in addresses: + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True) + + # Test importing of a P2PKH address via descriptor + key = get_key(self.nodes[0]) + self.log.info("Should import a p2pkh address from descriptor") + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "label": "Descriptor import test"}, + True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + solvable=True, + ismine=False, + label="Descriptor import test") + + # Test import fails if both desc and scriptPubKey are provided + key = get_key(self.nodes[0]) + self.log.info("Import should fail if both scriptPubKey and desc are provided") + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Both a descriptor and a scriptPubKey should not be provided.') + + # Test import fails if neither desc nor scriptPubKey are present + key = get_key(self.nodes[0]) + self.log.info("Import should fail if neither a descriptor nor a scriptPubKey are provided") + self.test_importmulti({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Either a descriptor or scriptPubKey must be provided.') + + # Test importing of a multisig via descriptor + key1 = get_key(self.nodes[0]) + key2 = get_key(self.nodes[0]) + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importmulti({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), + "timestamp": "now"}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(self.nodes[1], + key1.p2pkh_addr, + ismine=False, + iswatchonly=False) + + # Import pubkeys with key origin info + self.log.info("Addresses should have hd keypath and master key id after import with key origin") + pub_addr = self.nodes[1].getnewaddress() + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + pub_keypath = info['hdkeypath'] + pub_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : descsum_create("wpkh([" + pub_fpr + pub_keypath[1:] +"]" + pub + ")"), + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(pub_import_info['pubkey'], pub) + assert_equal(pub_import_info['hdkeypath'], pub_keypath) + + # Import privkeys with key origin info + priv_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(priv_addr) + priv = self.nodes[1].dumpprivkey(priv_addr) + priv_keypath = info['hdkeypath'] + priv_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : descsum_create("wpkh([" + priv_fpr + priv_keypath[1:] + "]" + priv + ")"), + "timestamp": "now", + }] + ) + assert result[0]['success'] + priv_import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(priv_import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(priv_import_info['hdkeypath'], priv_keypath) + + # Make sure the key origin info are still there after a restart + self.stop_nodes() + self.start_nodes() + import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(import_info['hdkeypath'], pub_keypath) + import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(import_info['hdkeypath'], priv_keypath) + + # Check legacy import does not import key origin info + self.log.info("Legacy imports don't have key origin info") + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + result = self.nodes[0].importmulti( + [{ + 'scriptPubKey': {'address': pub_addr}, + 'pubkeys': [pub], + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['pubkey'], pub) + assert 'hdmasterfingerprint' not in pub_import_info + assert 'hdkeypath' not in pub_import_info + + # Import some public keys to the keypool of a no privkey wallet + self.log.info("Adding pubkey to keypool of disableprivkey wallet") + self.nodes[1].createwallet(wallet_name="noprivkeys", disable_private_keys=True) + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + "timestamp": "now", + }, + { + 'desc': descsum_create('wpkh(' + pub2 + ')'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 2) + newaddr1 = wrpc.getnewaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getnewaddress() + assert_equal(addr2, newaddr2) + + # Import some public keys to the internal keypool of a no privkey wallet + self.log.info("Adding pubkey to internal keypool of disableprivkey wallet") + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + 'internal': True, + "timestamp": "now", }, - "timestamp": "now", - "redeemscript": bytes_to_hex_str(redeem_script), - "witnessscript": multi_sig_script['redeemScript'] - }]) - assert_equal(result[0]['success'], True) - address_assert = self.nodes[1].getaddressinfo(multi_sig_script['address']) - assert_equal(address_assert['solvable'], True) + { + 'desc': descsum_create('wpkh(' + pub2 + ')'), + 'keypool': True, + 'internal': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize_hd_internal"], 2) + newaddr1 = wrpc.getrawchangeaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getrawchangeaddress() + assert_equal(addr2, newaddr2) + + # Import a multisig and make sure the keys don't go into the keypool + self.log.info('Imported scripts with pubkeys should not have their pubkeys go into the keypool') + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wsh(multi(2,' + pub1 + ',' + pub2 + '))'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + + # Cannot import those pubkeys to keypool of wallet with privkeys + self.log.info("Pubkeys cannot be added to the keypool of a wallet with private keys") + wrpc = self.nodes[1].get_wallet_rpc("") + assert wrpc.getwalletinfo()['private_keys_enabled'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert_equal(result[0]['error']['code'], -8) + assert_equal(result[0]['error']['message'], "Keys can only be imported to the keypool when private keys are disabled") + + # Make sure ranged imports import keys in order + self.log.info('Key ranges should be imported in order') + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + assert_equal(wrpc.getwalletinfo()["private_keys_enabled"], False) + xpub = "tpubDAXcJ7s7ZwicqjprRaEWdPoHKrCS215qxGYxpusRLLmJuT69ZSicuGdSfyvyKpvUNYBW1s2U3NSrT6vrCYB9e6nZUEvrqnwXPF8ArTCRXMY" + addresses = [ + 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv', # m/0'/0'/0 + 'bcrt1q8vprchan07gzagd5e6v9wd7azyucksq2xc76k8', # m/0'/0'/1 + 'bcrt1qtuqdtha7zmqgcrr26n2rqxztv5y8rafjp9lulu', # m/0'/0'/2 + 'bcrt1qau64272ymawq26t90md6an0ps99qkrse58m640', # m/0'/0'/3 + 'bcrt1qsg97266hrh6cpmutqen8s4s962aryy77jp0fg0', # m/0'/0'/4 + ] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), + 'keypool': True, + 'timestamp': 'now', + 'range' : [0, 4], + }] + ) + for i in range(0, 5): + addr = wrpc.getnewaddress('', 'bech32') + assert_equal(addr, addresses[i]) if __name__ == '__main__': - ImportMultiTest ().main () + ImportMultiTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index ceb9709712..e3aeb61197 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" @@ -21,8 +21,7 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert_equal(wallet_info_old['hdseedid'], wallet_info_old['hdmasterkeyid']) - assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') @@ -30,9 +29,8 @@ class KeyPoolTest(BitcoinTestFramework): addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert_equal(wallet_info['hdseedid'], wallet_info['hdmasterkeyid']) - assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid']) - assert(addr_data['hdseedid'] == wallet_info['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -61,7 +59,7 @@ class KeyPoolTest(BitcoinTestFramework): addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) - assert(len(addr) == 6) + assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index f1a441c399..0014555ade 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. @@ -17,15 +17,14 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, - sync_blocks, ) class KeypoolRestoreTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 2 - self.extra_args = [[], ['-keypool=100']] + self.num_nodes = 4 + self.extra_args = [[], ['-keypool=100'], ['-keypool=100'], ['-keypool=100']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -40,32 +39,47 @@ class KeypoolRestoreTest(BitcoinTestFramework): shutil.copyfile(wallet_path, wallet_backup_path) self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 0, 2) + connect_nodes_bi(self.nodes, 0, 3) - self.log.info("Generate keys for wallet") - for _ in range(90): - addr_oldpool = self.nodes[1].getnewaddress() - for _ in range(20): - addr_extpool = self.nodes[1].getnewaddress() + for i, output_type in enumerate(["legacy", "p2sh-segwit", "bech32"]): - self.log.info("Send funds to wallet") - self.nodes[0].sendtoaddress(addr_oldpool, 10) - self.nodes[0].generate(1) - self.nodes[0].sendtoaddress(addr_extpool, 5) - self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.log.info("Generate keys for wallet with address type: {}".format(output_type)) + idx = i+1 + for _ in range(90): + addr_oldpool = self.nodes[idx].getnewaddress(address_type=output_type) + for _ in range(20): + addr_extpool = self.nodes[idx].getnewaddress(address_type=output_type) - self.log.info("Restart node with wallet backup") - self.stop_node(1) - shutil.copyfile(wallet_backup_path, wallet_path) - self.start_node(1, self.extra_args[1]) - connect_nodes_bi(self.nodes, 0, 1) - self.sync_all() + # Make sure we're creating the outputs we expect + address_details = self.nodes[idx].validateaddress(addr_extpool) + if i == 0: + assert not address_details["isscript"] and not address_details["iswitness"] + elif i == 1: + assert address_details["isscript"] and not address_details["iswitness"] + else: + assert not address_details["isscript"] and address_details["iswitness"] + + + self.log.info("Send funds to wallet") + self.nodes[0].sendtoaddress(addr_oldpool, 10) + self.nodes[0].generate(1) + self.nodes[0].sendtoaddress(addr_extpool, 5) + self.nodes[0].generate(1) + self.sync_blocks() + + self.log.info("Restart node with wallet backup") + self.stop_node(idx) + shutil.copyfile(wallet_backup_path, wallet_path) + self.start_node(idx, self.extra_args[idx]) + connect_nodes_bi(self.nodes, 0, idx) + self.sync_all() - self.log.info("Verify keypool is restored and balance is correct") - assert_equal(self.nodes[1].getbalance(), 15) - assert_equal(self.nodes[1].listtransactions()[0]['category'], "receive") - # Check that we have marked all keys up to the used keypool key as used - assert_equal(self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['hdkeypath'], "m/0'/0'/110'") + self.log.info("Verify keypool is restored and balance is correct") + assert_equal(self.nodes[idx].getbalance(), 15) + assert_equal(self.nodes[idx].listtransactions()[0]['category'], "receive") + # Check that we have marked all keys up to the used keypool key as used + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress())['hdkeypath'], "m/0'/0'/110'") if __name__ == '__main__': diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 011975e371..5e94068930 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" @@ -10,7 +10,6 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_raises_rpc_error, - sync_blocks, ) @@ -24,7 +23,7 @@ class ReceivedByTest(BitcoinTestFramework): def run_test(self): # Generate block to get out of IBD self.nodes[0].generate(1) - sync_blocks(self.nodes) + self.sync_blocks() # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 25ab222375..021a29d4ac 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listsincelast RPC.""" @@ -98,7 +98,8 @@ class ListSinceBlockTest (BitcoinTestFramework): self.nodes[2].generate(7) self.log.info('lastblockhash=%s' % (lastblockhash)) - self.sync_all([self.nodes[:2], self.nodes[2:]]) + self.sync_all(self.nodes[:2]) + self.sync_all(self.nodes[2:]) self.join_network() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 8ca0387268..997d6e702c 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" @@ -11,9 +11,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, assert_equal, - bytes_to_hex_str, hex_str_to_bytes, - sync_mempools, ) def tx_from_hex(hexstring): @@ -25,12 +23,13 @@ def tx_from_hex(hexstring): class ListTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.enable_mocktime() def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.nodes[0].generate(1) # Get out of IBD + self.sync_all() # Simple send, 0 to 1: txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() @@ -125,9 +124,9 @@ class ListTransactionsTest(BitcoinTestFramework): # 1. Chain a few transactions that don't opt-in. txid_1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - assert(not is_opt_in(self.nodes[0], txid_1)) + assert not is_opt_in(self.nodes[0], txid_1) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) # Tx2 will build off txid_1, still not opting in to RBF. @@ -145,9 +144,9 @@ class ListTransactionsTest(BitcoinTestFramework): txid_2 = self.nodes[1].sendrawtransaction(tx2_signed) # ...and check the result - assert(not is_opt_in(self.nodes[1], txid_2)) + assert not is_opt_in(self.nodes[1], txid_2) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) # Tx3 will opt-in to RBF @@ -157,13 +156,13 @@ class ListTransactionsTest(BitcoinTestFramework): tx3 = self.nodes[0].createrawtransaction(inputs, outputs) tx3_modified = tx_from_hex(tx3) tx3_modified.vin[0].nSequence = 0 - tx3 = bytes_to_hex_str(tx3_modified.serialize()) + tx3 = tx3_modified.serialize().hex() tx3_signed = self.nodes[0].signrawtransactionwithwallet(tx3)['hex'] txid_3 = self.nodes[0].sendrawtransaction(tx3_signed) - assert(is_opt_in(self.nodes[0], txid_3)) + assert is_opt_in(self.nodes[0], txid_3) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) # Tx4 will chain off tx3. Doesn't signal itself, but depends on one @@ -175,21 +174,21 @@ class ListTransactionsTest(BitcoinTestFramework): tx4_signed = self.nodes[1].signrawtransactionwithwallet(tx4)["hex"] txid_4 = self.nodes[1].sendrawtransaction(tx4_signed) - assert(not is_opt_in(self.nodes[1], txid_4)) + assert not is_opt_in(self.nodes[1], txid_4) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee - tx3_b = bytes_to_hex_str(tx3_b.serialize()) + tx3_b = tx3_b.serialize().hex() tx3_b_signed = self.nodes[0].signrawtransactionwithwallet(tx3_b)['hex'] - txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, True) - assert(is_opt_in(self.nodes[0], txid_3b)) + txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, 0) + assert is_opt_in(self.nodes[0], txid_3b) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) - sync_mempools(self.nodes) + self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) # Check gettransaction as well: @@ -202,7 +201,7 @@ class ListTransactionsTest(BitcoinTestFramework): # After mining a transaction, it's no longer BIP125-replaceable self.nodes[0].generate(1) - assert(txid_3b not in self.nodes[0].getrawmempool()) + assert txid_3b not in self.nodes[0].getrawmempool() assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 8ab569a3c3..984ffab5a4 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. @@ -315,6 +315,14 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], True) + # Test .walletlock file is closed + self.start_node(1) + wallet = os.path.join(self.options.tmpdir, 'my_wallet') + self.nodes[0].createwallet(wallet) + assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet) + self.nodes[0].unloadwallet(wallet) + self.nodes[1].loadwallet(wallet) + if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 00bf58d709..5810e94938 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -1,32 +1,77 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test resendwallettransactions RPC.""" +"""Test that the wallet resends transactions periodically.""" +from collections import defaultdict +import time +from test_framework.blocktools import create_block, create_coinbase +from test_framework.messages import ToHex +from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, wait_until + +class P2PStoreTxInvs(P2PInterface): + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == 1: + # save txid + self.tx_invs_received[i.hash] += 1 class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['--walletbroadcast=false']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): - # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. - assert_raises_rpc_error(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + node = self.nodes[0] # alias + + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a new transaction and wait until it's broadcast") + txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + + # Wallet rebroadcast is first scheduled 1 sec after startup (see + # nNextResend in ResendWalletTransactions()). Sleep for just over a + # second to be certain that it has been called before the first + # setmocktime call below. + time.sleep(1.1) + + # Can take a few seconds due to transaction trickling + wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + + # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + node.add_p2p_connection(P2PStoreTxInvs()) + + self.log.info("Create a block") + # Create and submit a block without the transaction. + # Transactions are only rebroadcast if there has been a block at least five minutes + # after the last time we tried to broadcast. Use mocktime and give an extra minute to be sure. + block_time = int(time.time()) + 6 * 60 + node.setmocktime(block_time) + block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockchaininfo()['blocks']), block_time) + block.nVersion = 3 + block.rehash() + block.solve() + node.submitblock(ToHex(block)) - # Should return an empty array if there aren't unconfirmed wallet transactions. - self.stop_node(0) - self.start_node(0, extra_args=[]) - assert_equal(self.nodes[0].resendwallettransactions(), []) + # Transaction should not be rebroadcast + node.p2ps[1].sync_with_ping() + assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - # Should return an array with the unconfirmed wallet transaction. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - assert_equal(self.nodes[0].resendwallettransactions(), [txid]) + self.log.info("Transaction should be rebroadcast after 30 minutes") + # Use mocktime and give an extra 5 minutes to be sure. + rebroadcast_time = int(time.time()) + 41 * 60 + node.setmocktime(rebroadcast_time) + wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) if __name__ == '__main__': ResendWalletTransactionsTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index d78c105c17..60d7205887 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" +import io from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, disconnect_nodes, - sync_blocks, ) +from test_framework.messages import CTransaction, COIN class TxnMallTest(BitcoinTestFramework): def set_test_params(self): @@ -65,32 +66,27 @@ class TxnMallTest(BitcoinTestFramework): # Construct a clone of tx1, to be malleated rawtx1 = self.nodes[0].getrawtransaction(txid1, 1) - clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"]}] + clone_inputs = [{"txid": rawtx1["vin"][0]["txid"], "vout": rawtx1["vin"][0]["vout"], "sequence": rawtx1["vin"][0]["sequence"]}] clone_outputs = {rawtx1["vout"][0]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][0]["value"], rawtx1["vout"][1]["scriptPubKey"]["addresses"][0]: rawtx1["vout"][1]["value"]} clone_locktime = rawtx1["locktime"] clone_raw = self.nodes[0].createrawtransaction(clone_inputs, clone_outputs, clone_locktime) # createrawtransaction randomizes the order of its outputs, so swap them if necessary. - # output 0 is at version+#inputs+input+sigstub+sequence+#outputs - # 40 BTC serialized is 00286bee00000000 - pos0 = 2 * (4 + 1 + 36 + 1 + 4 + 1) - hex40 = "00286bee00000000" - output_len = 16 + 2 + 2 * int("0x" + clone_raw[pos0 + 16:pos0 + 16 + 2], 0) - if (rawtx1["vout"][0]["value"] == 40 and clone_raw[pos0:pos0 + 16] != hex40 or rawtx1["vout"][0]["value"] != 40 and clone_raw[pos0:pos0 + 16] == hex40): - output0 = clone_raw[pos0:pos0 + output_len] - output1 = clone_raw[pos0 + output_len:pos0 + 2 * output_len] - clone_raw = clone_raw[:pos0] + output1 + output0 + clone_raw[pos0 + 2 * output_len:] + clone_tx = CTransaction() + clone_tx.deserialize(io.BytesIO(bytes.fromhex(clone_raw))) + if (rawtx1["vout"][0]["value"] == 40 and clone_tx.vout[0].nValue != 40*COIN or rawtx1["vout"][0]["value"] != 40 and clone_tx.vout[0].nValue == 40*COIN): + (clone_tx.vout[0], clone_tx.vout[1]) = (clone_tx.vout[1], clone_tx.vout[0]) # Use a different signature hash type to sign. This creates an equivalent but malleated clone. # Don't send the clone anywhere yet - tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_raw, None, "ALL|ANYONECANPAY") + tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_tx.serialize().hex(), None, "ALL|ANYONECANPAY") assert_equal(tx1_clone["complete"], True) # Have node0 mine a block, if requested: if (self.options.mine_block): self.nodes[0].generate(1) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -126,7 +122,7 @@ class TxnMallTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(node0_tx2["hex"]) self.nodes[2].sendrawtransaction(tx2["hex"]) self.nodes[2].generate(1) # Mine another block to make sure we sync - sync_blocks(self.nodes) + self.sync_blocks() # Re-fetch transaction info: tx1 = self.nodes[0].gettransaction(txid1) diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index f114d5ab68..40eeb4048c 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" @@ -11,7 +11,6 @@ from test_framework.util import ( connect_nodes, disconnect_nodes, find_output, - sync_blocks, ) class TxnMallTest(BitcoinTestFramework): @@ -34,6 +33,14 @@ class TxnMallTest(BitcoinTestFramework): def run_test(self): # All nodes should start with 1,250 BTC: starting_balance = 1250 + + # All nodes should be out of IBD. + # If the nodes are not all out of IBD, that can interfere with + # blockchain sync later in the test when nodes are connected, due to + # timing issues. + for n in self.nodes: + assert n.getblockchaininfo()["initialblockdownload"] == False + for i in range(4): assert_equal(self.nodes[i].getbalance(), starting_balance) self.nodes[i].getnewaddress("") # bug workaround, coins generated assigned to first getnewaddress! @@ -78,7 +85,7 @@ class TxnMallTest(BitcoinTestFramework): # Have node0 mine a block: if (self.options.mine_block): self.nodes[0].generate(1) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -111,7 +118,7 @@ class TxnMallTest(BitcoinTestFramework): # Reconnect the split network, and sync chain: connect_nodes(self.nodes[1], 2) self.nodes[2].generate(1) # Mine another block to make sure we sync - sync_blocks(self.nodes) + self.sync_blocks() assert_equal(self.nodes[0].gettransaction(doublespend_txid)["confirmations"], 2) # Re-fetch transaction info: diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py new file mode 100755 index 0000000000..1869f71753 --- /dev/null +++ b/test/fuzz/test_runner.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Run fuzz test targets. +""" + +import argparse +import configparser +import os +import sys +import subprocess +import logging + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "-l", + "--loglevel", + dest="loglevel", + default="INFO", + help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", + ) + parser.add_argument( + '--export_coverage', + action='store_true', + help='If true, export coverage information to files in the seed corpus', + ) + parser.add_argument( + 'seed_dir', + help='The seed corpus to run on (must contain subfolders for each fuzz target).', + ) + parser.add_argument( + 'target', + nargs='*', + help='The target(s) to run. Default is to run all targets.', + ) + + args = parser.parse_args() + + # Set up logging + logging.basicConfig( + format='%(message)s', + level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), + ) + + # Read config generated by configure. + config = configparser.ConfigParser() + configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" + config.read_file(open(configfile, encoding="utf8")) + + if not config["components"].getboolean("ENABLE_FUZZ"): + logging.error("Must have fuzz targets built") + sys.exit(1) + + # Build list of tests + test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include')) + + if not test_list_all: + logging.error("No fuzz targets found") + sys.exit(1) + + logging.info("Fuzz targets found: {}".format(test_list_all)) + + args.target = args.target or test_list_all # By default run all + test_list_error = list(set(args.target).difference(set(test_list_all))) + if test_list_error: + logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) + test_list_selection = list(set(test_list_all).intersection(set(args.target))) + if not test_list_selection: + logging.error("No fuzz targets selected") + logging.info("Fuzz targets selected: {}".format(test_list_selection)) + + try: + help_output = subprocess.run( + args=[ + os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), + '-help=1', + ], + timeout=1, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stderr + if "libFuzzer" not in help_output: + logging.error("Must be built with libFuzzer") + sys.exit(1) + except subprocess.TimeoutExpired: + logging.error("subprocess timed out: Currently only libFuzzer is supported") + sys.exit(1) + + run_once( + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + export_coverage=args.export_coverage, + ) + + +def run_once(*, corpus, test_list, build_dir, export_coverage): + for t in test_list: + args = [ + os.path.join(build_dir, 'src', 'test', 'fuzz', t), + '-runs=1', + os.path.join(corpus, t), + ] + logging.debug('Run {} with args {}'.format(t, args)) + output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug('Output: {}'.format(output)) + if not export_coverage: + continue + for l in output.splitlines(): + if 'INITED' in l: + with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file: + cov_file.write(l) + break + + +def parse_test_list(makefile): + with open(makefile, encoding='utf-8') as makefile_test: + test_list_all = [] + read_targets = False + for line in makefile_test.readlines(): + line = line.strip().replace('test/fuzz/', '').replace(' \\', '') + if read_targets: + if not line: + break + test_list_all.append(line) + continue + + if line == 'FUZZ_TARGETS =': + read_targets = True + return test_list_all + + +if __name__ == '__main__': + main() diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index b0d9f87958..3b05d5055c 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,8 +26,12 @@ SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def main(): - used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') - docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + if sys.version_info >= (3, 6): + used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') + docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + else: + used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() + docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index f1327469f3..4267f9fa0d 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -20,23 +20,23 @@ fi RET=0 PREV_BRANCH=`git name-rev --name-only HEAD` PREV_HEAD=`git rev-parse HEAD` -for i in `git rev-list --reverse $1`; do - if git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:"; then - git checkout --quiet $i^ || exit - SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" +for commit in `git rev-list --reverse $1`; do + if git rev-list -n 1 --pretty="%s" $commit | grep -q "^scripted-diff:"; then + git checkout --quiet $commit^ || exit + SCRIPT="`git rev-list --format=%b -n1 $commit | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" if test "x$SCRIPT" = "x"; then - echo "Error: missing script for: $i" + echo "Error: missing script for: $commit" echo "Failed" RET=1 else - echo "Running script for: $i" + echo "Running script for: $commit" echo "$SCRIPT" - eval "$SCRIPT" - git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1 + (eval "$SCRIPT") + git --no-pager diff --exit-code $commit && echo "OK" || (echo "Failed"; false) || RET=1 fi git reset --quiet --hard HEAD else - if git rev-list "--format=%b" -n1 $i | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then + if git rev-list "--format=%b" -n1 $commit | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then echo "Error: script block marker but no scripted-diff in title" echo "Failed" RET=1 diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index be67cbed31..e1a99abc49 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -10,10 +10,9 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" - "checkpoints -> validation -> checkpoints" "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" - "policy/policy -> validation -> policy/policy" + "policy/policy -> policy/settings -> policy/policy" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bantablemodel -> qt/clientmodel -> qt/bantablemodel" "qt/bitcoingui -> qt/utilitydialog -> qt/bitcoingui" @@ -30,14 +29,13 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "wallet/coincontrol -> wallet/wallet -> wallet/coincontrol" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "policy/fees -> policy/policy -> validation -> policy/fees" + "policy/fees -> txmempool -> validation -> policy/fees" "policy/rbf -> txmempool -> validation -> policy/rbf" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/addressbookpage" "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/guiutil" "txmempool -> validation -> validationinterface -> txmempool" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/receivecoinsdialog -> qt/addressbookpage" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/signverifymessagedialog -> qt/addressbookpage" - "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/intro -> qt/guiutil" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/sendcoinsdialog -> qt/sendcoinsentry -> qt/addressbookpage" ) diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 5caebf3739..224e62f04a 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -39,7 +39,7 @@ def parse_function_calls(function_name, source_code): >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ - assert(type(function_name) is str and type(source_code) is str and function_name) + assert type(function_name) is str and type(source_code) is str and function_name lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] @@ -53,7 +53,7 @@ def normalize(s): >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ - assert(type(s) is str) + assert type(s) is str s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub("/\*.*?\*/", " ", s) @@ -77,7 +77,7 @@ def escape(s): >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s @@ -92,7 +92,7 @@ def unescape(s): >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s @@ -151,10 +151,10 @@ def parse_function_call_and_arguments(function_name, function_call): >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] """ - assert(type(function_name) is str and type(function_call) is str and function_name) + assert type(function_name) is str and type(function_call) is str and function_name remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) - assert(remaining.startswith(expected_function_call)) + assert remaining.startswith(expected_function_call) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 @@ -213,7 +213,7 @@ def parse_string_content(argument): >>> parse_string_content('1 2 3') '' """ - assert(type(argument) is str) + assert type(argument) is str string_content = "" in_string = False for char in normalize(escape(argument)): @@ -240,13 +240,12 @@ def count_format_specifiers(format_string): >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ - assert(type(format_string) is str) + assert type(format_string) is str + format_string = format_string.replace('%%', 'X') n = 0 in_specifier = False for i, char in enumerate(format_string): - if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%": - pass - elif char == "%": + if char == "%": in_specifier = True n += 1 elif char in "aAcdeEfFgGinopsuxX": @@ -263,27 +262,27 @@ def main(): parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " "argument (e.g. 1 in the case of fprintf)", default=0) parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", type=argparse.FileType("r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args() - exit_code = 0 - for f in args.file: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue sys.exit(exit_code) diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index 2c443abf6b..c994ae3f4d 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -11,20 +11,20 @@ export LC_ALL=C FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - FatalError,0 - fprintf,1 - LogConnectFailure,1 - LogPrint,1 - LogPrintf,0 - printf,0 - snprintf,2 - sprintf,1 - strprintf,0 - vfprintf,1 - vprintf,1 - vsnprintf,1 - vsprintf,1 - WalletLogPrintf,0 + "FatalError,0" + "fprintf,1" + "LogConnectFailure,1" + "LogPrint,1" + "LogPrintf,0" + "printf,0" + "snprintf,2" + "sprintf,1" + "strprintf,0" + "vfprintf,1" + "vprintf,1" + "vsnprintf,1" + "vsprintf,1" + "WalletLogPrintf,0" ) EXIT_CODE=0 diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index f6d0fd382b..4b9e2615b6 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -50,7 +50,6 @@ EXPECTED_BOOST_INCLUDES=( boost/algorithm/string/classification.hpp boost/algorithm/string/replace.hpp boost/algorithm/string/split.hpp - boost/bind.hpp boost/chrono/chrono.hpp boost/date_time/posix_time/posix_time.hpp boost/filesystem.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 44170a6b5a..2b6c78c2c8 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -4,30 +4,21 @@ export LC_ALL=C KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" "src/bitcoin-tx.cpp.*trim_right" - "src/bitcoin-tx.cpp:.*atoi" - "src/core_read.cpp.*is_digit" "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/init.cpp:.*fprintf" "src/qt/rpcconsole.cpp:.*atoi" - "src/qt/rpcconsole.cpp:.*isdigit" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/getarg_tests.cpp.*split" "src/torcontrol.cpp:.*atoi" "src/torcontrol.cpp:.*strtol" - "src/uint256.cpp:.*tolower" - "src/util/system.cpp:.*atoi" - "src/util/system.cpp:.*fprintf" - "src/util/system.cpp:.*tolower" - "src/util/moneystr.cpp:.*isdigit" "src/util/strencodings.cpp:.*atoi" "src/util/strencodings.cpp:.*strtol" - "src/util/strencodings.cpp:.*strtoll" "src/util/strencodings.cpp:.*strtoul" - "src/util/strencodings.cpp:.*strtoull" "src/util/strencodings.h:.*atoi" + "src/util/system.cpp:.*atoi" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh index 3341f794f9..588ba428d7 100755 --- a/test/lint/lint-python-dead-code.sh +++ b/test/lint/lint-python-dead-code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (c) 2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying @@ -15,5 +15,5 @@ fi vulture \ --min-confidence 60 \ - --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,get_ecdh_key,get_privkey,is_compressed,is_fullyvalid,msg_generic,on_*,optionxform,restype,set_privkey" \ - $(git ls-files -- "*.py" ":(exclude)contrib/") + --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,is_compressed,is_valid,verify_ecdsa,msg_generic,on_*,optionxform,restype,profile_with_perf" \ + $(git ls-files -- "*.py" ":(exclude)contrib/" ":(exclude)test/functional/data/invalid_txs.py") diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index d44a585294..f5b851aeab 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying @@ -8,76 +8,79 @@ export LC_ALL=C -# E101 indentation contains mixed spaces and tabs -# E112 expected an indented block -# E113 unexpected indentation -# E115 expected an indented block (comment) -# E116 unexpected indentation (comment) -# E125 continuation line with same indent as next logical line -# E129 visually indented line with same indent as next logical line -# E131 continuation line unaligned for hanging indent -# E133 closing bracket is missing indentation -# E223 tab before operator -# E224 tab after operator -# E242 tab after ',' -# E266 too many leading '#' for block comment -# E271 multiple spaces after keyword -# E272 multiple spaces before keyword -# E273 tab after keyword -# E274 tab before keyword -# E275 missing whitespace after keyword -# E304 blank lines found after function decorator -# E306 expected 1 blank line before a nested definition -# E401 multiple imports on one line -# E402 module level import not at top of file -# F403 'from foo_module import *' used; unable to detect undefined names -# F405 foo_function may be undefined, or defined from star imports: bar_module -# E502 the backslash is redundant between brackets -# E701 multiple statements on one line (colon) -# E702 multiple statements on one line (semicolon) -# E703 statement ends with a semicolon -# E714 test for object identity should be "is not" -# E721 do not compare types, use "isinstance()" -# E741 do not use variables named "l", "O", or "I" -# E742 do not define classes named "l", "O", or "I" -# E743 do not define functions named "l", "O", or "I" -# E901 SyntaxError: invalid syntax -# E902 TokenError: EOF in multi-line string -# F401 module imported but unused -# F402 import module from line N shadowed by loop variable -# F404 future import(s) name after other statements -# F406 "from module import *" only allowed at module level -# F407 an undefined __future__ feature name was imported -# F601 dictionary key name repeated with different values -# F602 dictionary key variable name repeated with different values -# F621 too many expressions in an assignment with star-unpacking -# F622 two or more starred expressions in an assignment (a, *b, *c = d) -# F631 assertion test is a tuple, which are always True -# F701 a break statement outside of a while or for loop -# F702 a continue statement outside of a while or for loop -# F703 a continue statement in a finally block in a loop -# F704 a yield or yield from statement outside of a function -# F705 a return statement with arguments inside a generator -# F706 a return statement outside of a function/method -# F707 an except: block as not the last exception handler -# F811 redefinition of unused name from line N -# F812 list comprehension redefines 'foo' from line N -# F821 undefined name 'Foo' -# F822 undefined name name in __all__ -# F823 local variable name … referenced before assignment -# F831 duplicate argument name in function definition -# F841 local variable 'foo' is assigned to but never used -# W191 indentation contains tabs -# W291 trailing whitespace -# W292 no newline at end of file -# W293 blank line contains whitespace -# W504 line break after binary operator -# W601 .has_key() is deprecated, use "in" -# W602 deprecated form of raising exception -# W603 "<>" is deprecated, use "!=" -# W604 backticks are deprecated, use "repr()" -# W605 invalid escape sequence "x" -# W606 'async' and 'await' are reserved keywords starting with Python 3.7 +enabled=( + E101 # indentation contains mixed spaces and tabs + E112 # expected an indented block + E113 # unexpected indentation + E115 # expected an indented block (comment) + E116 # unexpected indentation (comment) + E125 # continuation line with same indent as next logical line + E129 # visually indented line with same indent as next logical line + E131 # continuation line unaligned for hanging indent + E133 # closing bracket is missing indentation + E223 # tab before operator + E224 # tab after operator + E242 # tab after ',' + E266 # too many leading '#' for block comment + E271 # multiple spaces after keyword + E272 # multiple spaces before keyword + E273 # tab after keyword + E274 # tab before keyword + E275 # missing whitespace after keyword + E304 # blank lines found after function decorator + E306 # expected 1 blank line before a nested definition + E401 # multiple imports on one line + E402 # module level import not at top of file + E502 # the backslash is redundant between brackets + E701 # multiple statements on one line (colon) + E702 # multiple statements on one line (semicolon) + E703 # statement ends with a semicolon + E711 # comparison to None should be 'if cond is None:' + E714 # test for object identity should be "is not" + E721 # do not compare types, use "isinstance()" + E741 # do not use variables named "l", "O", or "I" + E742 # do not define classes named "l", "O", or "I" + E743 # do not define functions named "l", "O", or "I" + E901 # SyntaxError: invalid syntax + E902 # TokenError: EOF in multi-line string + F401 # module imported but unused + F402 # import module from line N shadowed by loop variable + F403 # 'from foo_module import *' used; unable to detect undefined names + F404 # future import(s) name after other statements + F405 # foo_function may be undefined, or defined from star imports: bar_module + F406 # "from module import *" only allowed at module level + F407 # an undefined __future__ feature name was imported + F601 # dictionary key name repeated with different values + F602 # dictionary key variable name repeated with different values + F621 # too many expressions in an assignment with star-unpacking + F622 # two or more starred expressions in an assignment (a, *b, *c = d) + F631 # assertion test is a tuple, which are always True + F701 # a break statement outside of a while or for loop + F702 # a continue statement outside of a while or for loop + F703 # a continue statement in a finally block in a loop + F704 # a yield or yield from statement outside of a function + F705 # a return statement with arguments inside a generator + F706 # a return statement outside of a function/method + F707 # an except: block as not the last exception handler + F811 # redefinition of unused name from line N + F812 # list comprehension redefines 'foo' from line N + F821 # undefined name 'Foo' + F822 # undefined name name in __all__ + F823 # local variable name … referenced before assignment + F831 # duplicate argument name in function definition + F841 # local variable 'foo' is assigned to but never used + W191 # indentation contains tabs + W291 # trailing whitespace + W292 # no newline at end of file + W293 # blank line contains whitespace + W504 # line break after binary operator + W601 # .has_key() is deprecated, use "in" + W602 # deprecated form of raising exception + W603 # "<>" is deprecated, use "!=" + W604 # backticks are deprecated, use "repr()" + W605 # invalid escape sequence "x" + W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 +) if ! command -v flake8 > /dev/null; then echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\"" @@ -87,4 +90,4 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then exit 0 fi -PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 "${@:-.}" +PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") "${@:-.}" diff --git a/test/lint/lint-python-shebang.sh b/test/lint/lint-shebang.sh index 4ff87f0bf7..fda22592d3 100755 --- a/test/lint/lint-python-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Shebang must use python3 (not python or python2) +# Assert expected shebang lines export LC_ALL=C EXIT_CODE=0 @@ -10,4 +10,11 @@ for PYTHON_FILE in $(git ls-files -- "*.py"); do EXIT_CODE=1 fi done +for SHELL_FILE in $(git ls-files -- "*.sh"); do + if [[ $(head -n 1 "${SHELL_FILE}") != "#!/usr/bin/env bash" && + $(head -n 1 "${SHELL_FILE}") != "#!/bin/sh" ]]; then + echo "Missing expected shebang \"#!/usr/bin/env bash\" or \"#!/bin/sh\" in ${SHELL_FILE}" + EXIT_CODE=1 + fi +done exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 9af3c10ed6..6f5e6546c5 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -13,7 +13,7 @@ export LC_ALL=C # respectively. So export LC_ALL=C is set as required by lint-shell-locale.sh # but unset here in case of running in Travis. if [ "$TRAVIS" = "true" ]; then - unset LC_ALL + unset LC_ALL fi if ! command -v shellcheck > /dev/null; then @@ -22,26 +22,26 @@ if ! command -v shellcheck > /dev/null; then fi # Disabled warnings: -# SC1087: Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet). -# SC1117: Backslash is literal in "\.". Prefer explicit escaping: "\\.". -# SC2001: See if you can use ${variable//search/replace} instead. -# SC2004: $/${} is unnecessary on arithmetic variables. -# SC2005: Useless echo? Instead of 'echo $(cmd)', just use 'cmd'. -# SC2006: Use $(..) instead of legacy `..`. -# SC2016: Expressions don't expand in single quotes, use double quotes for that. -# SC2028: echo won't expand escape sequences. Consider printf. -# SC2046: Quote this to prevent word splitting. -# SC2048: Use "$@" (with quotes) to prevent whitespace problems. -# SC2066: Since you double quoted this, it will not word split, and the loop will only run once. -# SC2086: Double quote to prevent globbing and word splitting. -# SC2116: Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'. -# SC2148: Tips depend on target shell and yours is unknown. Add a shebang. -# SC2162: read without -r will mangle backslashes. -# SC2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined. -# SC2166: Prefer [ p ] || [ q ] as [ p -o q ] is not well defined. -# SC2181: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. -# SC2206: Quote to prevent word splitting, or split robustly with mapfile or read -a. -# SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). -# SC2230: which is non-standard. Use builtin 'command -v' instead. -shellcheck -e SC1087,SC1117,SC2001,SC2004,SC2005,SC2006,SC2016,SC2028,SC2046,SC2048,SC2066,SC2086,SC2116,SC2148,SC2162,SC2166,SC2181,SC2206,SC2207,SC2230 \ +disabled=( + SC1087 # Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet). + SC2001 # See if you can use ${variable//search/replace} instead. + SC2004 # $/${} is unnecessary on arithmetic variables. + SC2005 # Useless echo? Instead of 'echo $(cmd)', just use 'cmd'. + SC2006 # Use $(..) instead of legacy `..`. + SC2016 # Expressions don't expand in single quotes, use double quotes for that. + SC2028 # echo won't expand escape sequences. Consider printf. + SC2046 # Quote this to prevent word splitting. + SC2048 # Use "$@" (with quotes) to prevent whitespace problems. + SC2066 # Since you double quoted this, it will not word split, and the loop will only run once. + SC2086 # Double quote to prevent globbing and word splitting. + SC2116 # Useless echo? Instead of 'cmd $(echo foo)', just use 'cmd foo'. + SC2162 # read without -r will mangle backslashes. + SC2166 # Prefer [ p ] {&&,||} [ q ] as [ p -{a,o} q ] is not well defined. + SC2181 # Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. + SC2206 # Quote to prevent word splitting, or split robustly with mapfile or read -a. + SC2207 # Prefer mapfile or read -a to split command output (or quote to avoid splitting). + SC2230 # which is non-standard. Use builtin 'command -v' instead. + SC2236 # Don't force -n instead of ! -z. +) +shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \ $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/') diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index beb7ec42f4..d5c1dee42d 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -12,18 +12,18 @@ export LC_ALL=C while getopts "?" opt; do case $opt in ?) - echo "Usage: .lint-whitespace.sh [N]" - echo " TRAVIS_COMMIT_RANGE='<commit range>' .lint-whitespace.sh" - echo " .lint-whitespace.sh -?" + echo "Usage: $0 [N]" + echo " TRAVIS_COMMIT_RANGE='<commit range>' $0" + echo " $0 -?" echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' .lint-whitespace.sh" + echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' $0" exit 0 ;; esac done if [ -z "${TRAVIS_COMMIT_RANGE}" ]; then - if [ "$1" ]; then + if [ -n "$1" ]; then TRAVIS_COMMIT_RANGE="HEAD~$1...HEAD" else TRAVIS_COMMIT_RANGE="HEAD" diff --git a/test/sanitizer_suppressions/lsan b/test/sanitizer_suppressions/lsan new file mode 100644 index 0000000000..6f15c0f1d4 --- /dev/null +++ b/test/sanitizer_suppressions/lsan @@ -0,0 +1,6 @@ +# Suppress warnings triggered in dependencies +leak:libcrypto +leak:libqminimal +leak:libQt5Core +leak:libQt5Gui +leak:libQt5Widgets diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 209c46f096..70eea34363 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -1,9 +1,6 @@ # ThreadSanitizer suppressions # ============================ -# fChecked is theoretically racy, practically only in unit tests -race:CheckBlock - # WalletBatch (unidentified deadlock) deadlock:WalletBatch @@ -14,11 +11,6 @@ deadlock:TestPotentialDeadLockDetected race:src/qt/test/* deadlock:src/qt/test/* -# WIP: Unidentified suppressions to run the functional tests -#race:zmqpublishnotifier.cpp -# -#deadlock:CreateWalletFromFile -#deadlock:importprivkey -#deadlock:walletdb.h -#deadlock:walletdb.cpp -#deadlock:wallet/db.cpp +# External libraries +deadlock:libdb +race:libzmq diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index e90d5c2ac0..643272de52 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -4,7 +4,6 @@ bool:wallet/wallet.cpp float-divide-by-zero:policy/fees.cpp float-divide-by-zero:validation.cpp float-divide-by-zero:wallet/wallet.cpp -nonnull-attribute:support/cleanse.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -16,6 +15,7 @@ unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/chacha20.cpp unsigned-integer-overflow:crypto/ctaes/ctaes.c +unsigned-integer-overflow:crypto/poly1305.cpp unsigned-integer-overflow:crypto/ripemd160.cpp unsigned-integer-overflow:crypto/sha1.cpp unsigned-integer-overflow:crypto/sha256.cpp @@ -29,7 +29,6 @@ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:stl_bvector.h -unsigned-integer-overflow:streams.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 92fef30e13..7b1cc2b031 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -9,14 +9,9 @@ Runs automatically during `make check`. Can also be run manually.""" -from __future__ import division,print_function,unicode_literals - import argparse import binascii -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import difflib import json import logging |