diff options
Diffstat (limited to 'test')
65 files changed, 2230 insertions, 1294 deletions
diff --git a/test/README.md b/test/README.md index 4dd512638d..868eb667ae 100644 --- a/test/README.md +++ b/test/README.md @@ -15,84 +15,172 @@ The util tests are run as part of `make check` target. The functional tests are run by the travis continuous build process whenever a pull request is opened. Both sets of tests can also be run locally. -Functional Test dependencies -============================ +# Running tests locally + +Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. + +### Functional tests + +#### Dependencies + The ZMQ functional test requires a python ZMQ library. To install it: - on Unix, run `sudo apt-get install python3-zmq` - on mac OS, run `pip3 install pyzmq` -Running tests locally -===================== +#### Running the tests -Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. +Individual tests can be run by directly calling the test script, eg: -Functional tests ----------------- +``` +test/functional/replace-by-fee.py +``` -You can run any single test by calling +or can be run through the test_runner harness, eg: - test/functional/test_runner.py <testname> +``` +test/functional/test_runner.py replace-by-fee.py +``` -Or you can run any combination (incl. duplicates) of tests by calling +You can run any combination (incl. duplicates) of tests by calling: - test/functional/test_runner.py <testname1> <testname2> <testname3> ... +``` +test/functional/test_runner.py <testname1> <testname2> <testname3> ... +``` -Run the regression test suite with +Run the regression test suite with: - test/functional/test_runner.py +``` +test/functional/test_runner.py +``` Run all possible tests with - test/functional/test_runner.py --extended +``` +test/functional/test_runner.py --extended +``` -By default, tests will be run in parallel. To specify how many jobs to run, -append `--jobs=n` (default n=4). +By default, up to 4 tests will be run in parallel by test_runner. To specify +how many jobs to run, append `--jobs=n` -If you want to create a basic coverage report for the RPC test suite, append `--coverage`. +The individual tests and the test_runner harness have many command-line +options. Run `test_runner.py -h` to see them all. -Possible options, which apply to each individual test run: +#### Troubleshooting and debugging test failures -``` - -h, --help show this help message and exit - --nocleanup Leave bitcoinds and test.* datadir on exit or error - --noshutdown Don't stop bitcoinds after the test execution - --srcdir=SRCDIR Source directory containing bitcoind/bitcoin-cli - (default: ../../src) - --tmpdir=TMPDIR Root directory for datadirs - --tracerpc Print out all RPC calls as they are made - --coveragedir=COVERAGEDIR - Write tested RPC commands into this directory +##### Resource contention + +The P2P and RPC ports used by the bitcoind nodes-under-test are chosen to make +conflicts with other processes unlikely. However, if there is another bitcoind +process running on the system (perhaps from a previous test which hasn't successfully +killed all its bitcoind nodes), then there may be a port conflict which will +cause the test to fail. It is recommended that you run the tests on a system +where no other bitcoind processes are running. + +On linux, the test_framework will warn if there is another +bitcoind process running when the tests are started. + +If there are zombie bitcoind processes after test failure, you can kill them +by running the following commands. **Note that these commands will kill all +bitcoind processes running on the system, so should not be used if any non-test +bitcoind processes are being run.** + +```bash +killall bitcoind ``` -If you set the environment variable `PYTHON_DEBUG=1` you will get some debug -output (example: `PYTHON_DEBUG=1 test/functional/test_runner.py wallet`). +or + +```bash +pkill -9 bitcoind +``` -A 200-block -regtest blockchain and wallets for four nodes -is created the first time a regression test is run and -is stored in the cache/ directory. Each node has 25 mature -blocks (25*50=1250 BTC) in its wallet. -After the first run, the cache/ blockchain and wallets are -copied into a temporary directory and used as the initial -test state. +##### Data directory cache -If you get into a bad state, you should be able -to recover with: +A pre-mined blockchain with 200 blocks is generated the first time a +functional test is run and is stored in test/cache. This speeds up +test startup times since new blockchains don't need to be generated for +each test. However, the cache may get into a bad state, in which case +tests will fail. If this happens, remove the cache directory (and make +sure bitcoind processes are stopped as above): ```bash rm -rf cache killall bitcoind ``` -Util tests ----------- +##### Test logging + +The tests contain logging at different levels (debug, info, warning, etc). By +default: + +- when run through the test_runner harness, *all* logs are written to + `test_framework.log` and no logs are output to the console. +- when run directly, *all* logs are written to `test_framework.log` and INFO + level and above are output to the console. +- when run on Travis, no logs are output to the console. However, if a test + fails, the `test_framework.log` and bitcoind `debug.log`s will all be dumped + to the console to help troubleshooting. + +To change the level of logs output to the console, use the `-l` command line +argument. + +`test_framework.log` and bitcoind `debug.log`s can be combined into a single +aggregate log by running the `combine_logs.py` script. The output can be plain +text, colorized text or html. For example: + +``` +combine_logs.py -c <test data directory> | less -r +``` + +will pipe the colorized logs from the test into less. + +Use `--tracerpc` to trace out all the RPC calls and responses to the console. For +some tests (eg any that use `submitblock` to submit a full block over RPC), +this can result in a lot of screen output. + +By default, the test data directory will be deleted after a successful run. +Use `--nocleanup` to leave the test data directory intact. The test data +directory is never deleted after a failed test. + +##### Attaching a debugger + +A python debugger can be attached to tests at any point. Just add the line: + +```py +import pdb; pdb.set_trace() +``` + +anywhere in the test. You will then be able to inspect variables, as well as +call methods that interact with the bitcoind nodes-under-test. + +If further introspection of the bitcoind instances themselves becomes +necessary, this can be accomplished by first setting a pdb breakpoint +at an appropriate location, running the test to that point, then using +`gdb` to attach to the process and debug. + +For instance, to attach to `self.node[1]` during a run: + +```bash +2017-06-27 14:13:56.686000 TestFramework (INFO): Initializing test directory /tmp/user/1000/testo9vsdjo3 +``` + +use the directory path to get the pid from the pid file: + +```bash +cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid +gdb /home/example/bitcoind <pid> +``` + +Note: gdb attach step may require `sudo` + +### Util tests Util tests can be run locally by running `test/util/bitcoin-util-test.py`. Use the `-v` option for verbose output. -Writing functional tests -======================== +# Writing functional tests You are encouraged to write functional tests for new or existing features. Further information about the functional test framework and individual diff --git a/test/functional/README.md b/test/functional/README.md index e6c4849702..44efda33d3 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -1,108 +1,154 @@ -Regression tests -================ +# Functional tests -### [test_framework/authproxy.py](test_framework/authproxy.py) -Taken from the [python-bitcoinrpc repository](https://github.com/jgarzik/python-bitcoinrpc). +### Writing Functional Tests -### [test_framework/test_framework.py](test_framework/test_framework.py) -Base class for new regression tests. +#### Example test -### [test_framework/util.py](test_framework/util.py) -Generally useful functions. +The [example_test.py](example_test.py) is a heavily commented example of a test case that uses both +the RPC and P2P interfaces. If you are writing your first test, copy that file +and modify to fit your needs. -### [test_framework/mininode.py](test_framework/mininode.py) -Basic code to support p2p connectivity to a bitcoind. +#### Coverage -### [test_framework/comptool.py](test_framework/comptool.py) -Framework for comparison-tool style, p2p tests. +Running `test_runner.py` with the `--coverage` argument tracks which RPCs are +called by the tests and prints a report of uncovered RPCs in the summary. This +can be used (along with the `--extended` argument) to find out which RPCs we +don't have test cases for. -### [test_framework/script.py](test_framework/script.py) -Utilities for manipulating transaction scripts (originally from python-bitcoinlib) +#### Style guidelines -### [test_framework/blockstore.py](test_framework/blockstore.py) -Implements disk-backed block and tx storage. +- Where possible, try to adhere to [PEP-8 guidelines]([https://www.python.org/dev/peps/pep-0008/) +- Use a python linter like flake8 before submitting PRs to catch common style + nits (eg trailing whitespace, unused imports, etc) +- Avoid wildcard imports where possible +- Use a module-level docstring to describe what the test is testing, and how it + is testing it. +- When subclassing the BitcoinTestFramwork, place overrides for the + `__init__()`, and `setup_xxxx()` methods at the top of the subclass, then + locally-defined helper methods, then the `run_test()` method. -### [test_framework/key.py](test_framework/key.py) -Wrapper around OpenSSL EC_Key (originally from python-bitcoinlib) +#### General test-writing advice -### [test_framework/bignum.py](test_framework/bignum.py) -Helpers for script.py +- Set `self.num_nodes` to the minimum number of nodes necessary for the test. + Having additional unrequired nodes adds to the execution time of the test as + well as memory/CPU/disk requirements (which is important when running tests in + parallel or on Travis). +- Avoid stop-starting the nodes multiple times during the test if possible. A + stop-start takes several seconds, so doing it several times blows up the + runtime of the test. +- Set the `self.setup_clean_chain` variable in `__init__()` to control whether + or not to use the cached data directories. The cached data directories + contain a 200-block pre-mined blockchain and wallets for four nodes. Each node + has 25 mature blocks (25x50=1250 BTC) in its wallet. +- When calling RPCs with lots of arguments, consider using named keyword + arguments instead of positional arguments to make the intent of the call + clear to readers. -### [test_framework/blocktools.py](test_framework/blocktools.py) -Helper functions for creating blocks and transactions. +#### RPC and P2P definitions -P2P test design notes ---------------------- +Test writers may find it helpful to refer to the definitions for the RPC and +P2P messages. These can be found in the following source files: -## Mininode +- `/src/rpc/*` for RPCs +- `/src/wallet/rpc*` for wallet RPCs +- `ProcessMessage()` in `/src/net_processing.cpp` for parsing P2P messages -* ```mininode.py``` contains all the definitions for objects that pass -over the network (```CBlock```, ```CTransaction```, etc, along with the network-level -wrappers for them, ```msg_block```, ```msg_tx```, etc). +#### Using the P2P interface -* P2P tests have two threads. One thread handles all network communication +- `mininode.py` contains all the definitions for objects that pass +over the network (`CBlock`, `CTransaction`, etc, along with the network-level +wrappers for them, `msg_block`, `msg_tx`, etc). + +- P2P tests have two threads. One thread handles all network communication with the bitcoind(s) being tested (using python's asyncore package); the other implements the test logic. -* ```NodeConn``` is the class used to connect to a bitcoind. If you implement -a callback class that derives from ```NodeConnCB``` and pass that to the -```NodeConn``` object, your code will receive the appropriate callbacks when +- `NodeConn` is the class used to connect to a bitcoind. If you implement +a callback class that derives from `NodeConnCB` and pass that to the +`NodeConn` object, your code will receive the appropriate callbacks when events of interest arrive. -* You can pass the same handler to multiple ```NodeConn```'s if you like, or pass -different ones to each -- whatever makes the most sense for your test. - -* Call ```NetworkThread.start()``` after all ```NodeConn``` objects are created to +- Call `NetworkThread.start()` after all `NodeConn` objects are created to start the networking thread. (Continue with the test logic in your existing thread.) -* RPC calls are available in p2p tests. +- Can be used to write tests where specific P2P protocol behavior is tested. +Examples tests are `p2p-accept-block.py`, `p2p-compactblocks.py`. -* Can be used to write free-form tests, where specific p2p-protocol behavior -is tested. Examples: ```p2p-accept-block.py```, ```p2p-compactblocks.py```. +#### Comptool -## Comptool +- Comptool is a Testing framework for writing tests that compare the block/tx acceptance +behavior of a bitcoind against 1 or more other bitcoind instances. It should not be used +to write static tests with known outcomes, since that type of test is easier to write and +maintain using the standard BitcoinTestFramework. -* Testing framework for writing tests that compare the block/tx acceptance -behavior of a bitcoind against 1 or more other bitcoind instances, or against -known outcomes, or both. - -* Set the ```num_nodes``` variable (defined in ```ComparisonTestFramework```) to start up -1 or more nodes. If using 1 node, then ```--testbinary``` can be used as a command line +- Set the `num_nodes` variable (defined in `ComparisonTestFramework`) to start up +1 or more nodes. If using 1 node, then `--testbinary` can be used as a command line option to change the bitcoind binary used by the test. If using 2 or more nodes, -then ```--refbinary``` can be optionally used to change the bitcoind that will be used +then `--refbinary` can be optionally used to change the bitcoind that will be used on nodes 2 and up. -* Implement a (generator) function called ```get_tests()``` which yields ```TestInstance```s. -Each ```TestInstance``` consists of: - - a list of ```[object, outcome, hash]``` entries - * ```object``` is a ```CBlock```, ```CTransaction```, or - ```CBlockHeader```. ```CBlock```'s and ```CTransaction```'s are tested for - acceptance. ```CBlockHeader```s can be used so that the test runner can deliver +- Implement a (generator) function called `get_tests()` which yields `TestInstance`s. +Each `TestInstance` consists of: + - A list of `[object, outcome, hash]` entries + * `object` is a `CBlock`, `CTransaction`, or + `CBlockHeader`. `CBlock`'s and `CTransaction`'s are tested for + acceptance. `CBlockHeader`s can be used so that the test runner can deliver complete headers-chains when requested from the bitcoind, to allow writing tests where blocks can be delivered out of order but still processed by headers-first bitcoind's. - * ```outcome``` is ```True```, ```False```, or ```None```. If ```True``` - or ```False```, the tip is compared with the expected tip -- either the + * `outcome` is `True`, `False`, or `None`. If `True` + or `False`, the tip is compared with the expected tip -- either the block passed in, or the hash specified as the optional 3rd entry. If - ```None``` is specified, then the test will compare all the bitcoind's + `None` is specified, then the test will compare all the bitcoind's being tested to see if they all agree on what the best tip is. - * ```hash``` is the block hash of the tip to compare against. Optional to + * `hash` is the block hash of the tip to compare against. Optional to specify; if left out then the hash of the block passed in will be used as the expected tip. This allows for specifying an expected tip while testing the handling of either invalid blocks or blocks delivered out of order, which complete a longer chain. - - ```sync_every_block```: ```True/False```. If ```False```, then all blocks + - `sync_every_block`: `True/False`. If `False`, then all blocks are inv'ed together, and the test runner waits until the node receives the last one, and tests only the last block for tip acceptance using the - outcome and specified tip. If ```True```, then each block is tested in + outcome and specified tip. If `True`, then each block is tested in sequence and synced (this is slower when processing many blocks). - - ```sync_every_transaction```: ```True/False```. Analogous to - ```sync_every_block```, except if the outcome on the last tx is "None", + - `sync_every_transaction`: `True/False`. Analogous to + `sync_every_block`, except if the outcome on the last tx is "None", then the contents of the entire mempool are compared across all bitcoind - connections. If ```True``` or ```False```, then only the last tx's + connections. If `True` or `False`, then only the last tx's acceptance is tested against the given outcome. -* For examples of tests written in this framework, see - ```invalidblockrequest.py``` and ```p2p-fullblocktest.py```. +- For examples of tests written in this framework, see + `invalidblockrequest.py` and `p2p-fullblocktest.py`. + +### test-framework modules + +#### [test_framework/authproxy.py](test_framework/authproxy.py) +Taken from the [python-bitcoinrpc repository](https://github.com/jgarzik/python-bitcoinrpc). + +#### [test_framework/test_framework.py](test_framework/test_framework.py) +Base class for functional tests. +#### [test_framework/util.py](test_framework/util.py) +Generally useful functions. + +#### [test_framework/mininode.py](test_framework/mininode.py) +Basic code to support P2P connectivity to a bitcoind. + +#### [test_framework/comptool.py](test_framework/comptool.py) +Framework for comparison-tool style, P2P tests. + +#### [test_framework/script.py](test_framework/script.py) +Utilities for manipulating transaction scripts (originally from python-bitcoinlib) + +#### [test_framework/blockstore.py](test_framework/blockstore.py) +Implements disk-backed block and tx storage. + +#### [test_framework/key.py](test_framework/key.py) +Wrapper around OpenSSL EC_Key (originally from python-bitcoinlib) + +#### [test_framework/bignum.py](test_framework/bignum.py) +Helpers for script.py + +#### [test_framework/blocktools.py](test_framework/blocktools.py) +Helper functions for creating blocks and transactions. diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index bb83042f35..7e5e4cf682 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -4,173 +4,162 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). -Connect to a single node. -Mine 2 (version 3) blocks (save the coinbases for later). -Generate 98 more version 3 blocks, verify the node accepts. -Mine 749 version 4 blocks, verify the node accepts. -Check that the new CLTV rules are not enforced on the 750th version 4 block. -Check that the new CLTV rules are enforced on the 751st version 4 block. -Mine 199 new version blocks. -Mine 1 old-version block. -Mine 1 new version block. -Mine 1 old version block, see that the node rejects. +Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height +1351. """ -from test_framework.test_framework import ComparisonTestFramework +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import * from test_framework.blocktools import create_coinbase, create_block -from test_framework.comptool import TestInstance, TestManager -from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP +from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP, CScriptNum from io import BytesIO -import time + +CLTV_HEIGHT = 1351 + +# Reject codes that we might receive in this test +REJECT_INVALID = 16 +REJECT_OBSOLETE = 17 +REJECT_NONSTANDARD = 64 def cltv_invalidate(tx): '''Modify the signature in vin 0 of the tx to fail CLTV Prepends -1 CLTV DROP in the scriptSig itself. + + TODO: test more ways that transactions using CLTV could be invalid (eg + locktime requirements fail, sequence time requirements fail, etc). ''' tx.vin[0].scriptSig = CScript([OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) - -class BIP65Test(ComparisonTestFramework): +def cltv_validate(node, tx, height): + '''Modify the signature in vin 0 of the tx to pass CLTV + Prepends <height> CLTV DROP in the scriptSig, and sets + the locktime to height''' + tx.vin[0].nSequence = 0 + tx.nLockTime = height + + # Need to re-sign, since nSequence and nLockTime changed + signed_result = node.signrawtransaction(ToHex(tx)) + new_tx = CTransaction() + new_tx.deserialize(BytesIO(hex_str_to_bytes(signed_result['hex']))) + + new_tx.vin[0].scriptSig = CScript([CScriptNum(height), OP_CHECKLOCKTIMEVERIFY, OP_DROP] + + list(CScript(new_tx.vin[0].scriptSig))) + return new_tx + +def create_transaction(node, coinbase, to_address, amount): + from_txid = node.getblock(coinbase)['tx'][0] + inputs = [{ "txid" : from_txid, "vout" : 0}] + outputs = { to_address : amount } + rawtx = node.createrawtransaction(inputs, outputs) + signresult = node.signrawtransaction(rawtx) + tx = CTransaction() + tx.deserialize(BytesIO(hex_str_to_bytes(signresult['hex']))) + return tx + +class BIP65Test(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=3']] + self.extra_args = [['-promiscuousmempoolflags=1', '-whitelist=127.0.0.1']] + self.setup_clean_chain = True def run_test(self): - test = TestManager(self, self.options.tmpdir) - test.add_all_connections(self.nodes) + node0 = NodeConnCB() + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) + node0.add_connection(connections[0]) + NetworkThread().start() # Start up network handling in another thread - test.run() - - def create_transaction(self, node, coinbase, to_address, amount): - from_txid = node.getblock(coinbase)['tx'][0] - inputs = [{ "txid" : from_txid, "vout" : 0}] - outputs = { to_address : amount } - rawtx = node.createrawtransaction(inputs, outputs) - signresult = node.signrawtransaction(rawtx) - tx = CTransaction() - f = BytesIO(hex_str_to_bytes(signresult['hex'])) - tx.deserialize(f) - return tx - - def get_tests(self): - - self.coinbase_blocks = self.nodes[0].generate(2) - height = 3 # height of the next block to build - self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) + + # wait_for_verack ensures that the P2P connection is fully up. + node0.wait_for_verack() + + self.log.info("Mining %d blocks", CLTV_HEIGHT - 2) + self.coinbase_blocks = self.nodes[0].generate(CLTV_HEIGHT - 2) self.nodeaddress = self.nodes[0].getnewaddress() - self.last_block_time = int(time.time()) - - ''' 398 more version 3 blocks ''' - test_blocks = [] - for i in range(398): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' Mine 749 version 4 blocks ''' - test_blocks = [] - for i in range(749): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' - Check that the new CLTV rules are not enforced in the 750th - version 3 block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[0], self.nodeaddress, 1.0) + + self.log.info("Test that an invalid-according-to-CLTV transaction can still appear in a block") + + spendtx = create_transaction(self.nodes[0], self.coinbase_blocks[0], + self.nodeaddress, 1.0) cltv_invalidate(spendtx) spendtx.rehash() - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 + tip = self.nodes[0].getbestblockhash() + block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 + block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time) + block.nVersion = 3 block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' Mine 199 new version blocks on last valid tip ''' - test_blocks = [] - for i in range(199): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' Mine 1 old version block ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + node0.send_and_ping(msg_block(block)) + assert_equal(self.nodes[0].getbestblockhash(), block.hash) + + self.log.info("Test that blocks must now be at least version 4") + tip = block.sha256 + block_time += 1 + block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time) block.nVersion = 3 - block.rehash() block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - ''' Mine 1 new version block ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + assert wait_until(lambda: "reject" in node0.last_message.keys()) + with mininode_lock: + assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000003)') + assert_equal(node0.last_message["reject"].data, block.sha256) + del node0.last_message["reject"] + + self.log.info("Test that invalid-according-to-cltv transactions cannot appear in a block") block.nVersion = 4 - block.rehash() - block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' - Check that the new CLTV rules are enforced in the 951st version 4 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) + + spendtx = create_transaction(self.nodes[0], self.coinbase_blocks[1], + self.nodeaddress, 1.0) cltv_invalidate(spendtx) spendtx.rehash() - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 + # First we show that this tx is valid except for CLTV by getting it + # accepted to the mempool (which we can achieve with + # -promiscuousmempoolflags). + node0.send_and_ping(msg_tx(spendtx)) + assert spendtx.hash in self.nodes[0].getrawmempool() + + # Now we verify that a block with this transaction is invalid. block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) - ''' Mine 1 old version block, should be invalid ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.rehash() + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) + + assert wait_until (lambda: "reject" in node0.last_message.keys()) + with mininode_lock: + assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(node0.last_message["reject"].data, block.sha256) + if node0.last_message["reject"].code == REJECT_INVALID: + # Generic rejection when a block is invalid + assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + else: + assert b'Negative locktime' in node0.last_message["reject"].reason + + self.log.info("Test that a version 4 block with a valid-according-to-CLTV transaction is accepted") + spendtx = cltv_validate(self.nodes[0], spendtx, CLTV_HEIGHT - 1) + spendtx.rehash() + + block.vtx.pop(1) + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) + + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + if __name__ == '__main__': BIP65Test().main() diff --git a/test/functional/bip65-cltv.py b/test/functional/bip65-cltv.py deleted file mode 100755 index ddf932c746..0000000000 --- a/test/functional/bip65-cltv.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the CHECKLOCKTIMEVERIFY (BIP65) soft-fork logic.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -class BIP65Test(BitcoinTestFramework): - def __init__(self): - super().__init__() - self.num_nodes = 3 - self.setup_clean_chain = False - self.extra_args = [[], ["-blockversion=3"], ["-blockversion=4"]] - - def setup_network(self): - self.setup_nodes() - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) - self.sync_all() - - def run_test(self): - cnt = self.nodes[0].getblockcount() - - # Mine some old-version blocks - self.nodes[1].generate(200) - cnt += 100 - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 100): - raise AssertionError("Failed to mine 100 version=3 blocks") - - # Mine 750 new-version blocks - for i in range(15): - self.nodes[2].generate(50) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 850): - raise AssertionError("Failed to mine 750 version=4 blocks") - - # TODO: check that new CHECKLOCKTIMEVERIFY rules are not enforced - - # Mine 1 new-version block - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 851): - raise AssertionError("Failed to mine a version=4 blocks") - - # TODO: check that new CHECKLOCKTIMEVERIFY rules are enforced - - # Mine 198 new-version blocks - for i in range(2): - self.nodes[2].generate(99) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1049): - raise AssertionError("Failed to mine 198 version=4 blocks") - - # Mine 1 old-version block - self.nodes[1].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1050): - raise AssertionError("Failed to mine a version=3 block after 949 version=4 blocks") - - # Mine 1 new-version blocks - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1051): - raise AssertionError("Failed to mine a version=4 block") - - # Mine 1 old-version blocks. This should fail - assert_raises_jsonrpc(-1,"CreateNewBlock: TestBlockValidity failed: bad-version(0x00000003)", self.nodes[1].generate, 1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1051): - raise AssertionError("Accepted a version=3 block after 950 version=4 blocks") - - # Mine 1 new-version blocks - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1052): - raise AssertionError("Failed to mine a version=4 block") - -if __name__ == '__main__': - BIP65Test().main() diff --git a/test/functional/bip9-softforks.py b/test/functional/bip9-softforks.py index b90b0ca628..f00232c9ff 100755 --- a/test/functional/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -15,6 +15,10 @@ mine a further 143 blocks (LOCKED_IN) test that enforcement has not triggered (which triggers ACTIVE) test that enforcement has triggered """ +from io import BytesIO +import shutil +import time +import itertools from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * @@ -22,9 +26,6 @@ from test_framework.mininode import CTransaction, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript, OP_1NEGATE, OP_CHECKSEQUENCEVERIFY, OP_DROP -from io import BytesIO -import time -import itertools class BIP9SoftForksTest(ComparisonTestFramework): diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index 31c7ebba90..38a9009544 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -4,28 +4,24 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). -Connect to a single node. -Mine 2 (version 2) blocks (save the coinbases for later). -Generate 98 more version 2 blocks, verify the node accepts. -Mine 749 version 3 blocks, verify the node accepts. -Check that the new DERSIG rules are not enforced on the 750th version 3 block. -Check that the new DERSIG rules are enforced on the 751st version 3 block. -Mine 199 new version blocks. -Mine 1 old-version block. -Mine 1 new version block. -Mine 1 old version block, see that the node rejects. +Test that the DERSIG soft-fork activates at (regtest) height 1251. """ -from test_framework.test_framework import ComparisonTestFramework +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import * from test_framework.blocktools import create_coinbase, create_block -from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript from io import BytesIO -import time -# A canonical signature consists of: +DERSIG_HEIGHT = 1251 + +# Reject codes that we might receive in this test +REJECT_INVALID = 16 +REJECT_OBSOLETE = 17 +REJECT_NONSTANDARD = 64 + +# A canonical signature consists of: # <30> <total len> <02> <len R> <R> <02> <len S> <S> <hashtype> def unDERify(tx): """ @@ -40,143 +36,122 @@ def unDERify(tx): else: newscript.append(i) tx.vin[0].scriptSig = CScript(newscript) - -class BIP66Test(ComparisonTestFramework): + +def create_transaction(node, coinbase, to_address, amount): + from_txid = node.getblock(coinbase)['tx'][0] + inputs = [{ "txid" : from_txid, "vout" : 0}] + outputs = { to_address : amount } + rawtx = node.createrawtransaction(inputs, outputs) + signresult = node.signrawtransaction(rawtx) + tx = CTransaction() + tx.deserialize(BytesIO(hex_str_to_bytes(signresult['hex']))) + return tx + +class BIP66Test(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 + self.extra_args = [['-promiscuousmempoolflags=1', '-whitelist=127.0.0.1']] + self.setup_clean_chain = True def run_test(self): - test = TestManager(self, self.options.tmpdir) - test.add_all_connections(self.nodes) + node0 = NodeConnCB() + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) + node0.add_connection(connections[0]) NetworkThread().start() # Start up network handling in another thread - test.run() - - def create_transaction(self, node, coinbase, to_address, amount): - from_txid = node.getblock(coinbase)['tx'][0] - inputs = [{ "txid" : from_txid, "vout" : 0}] - outputs = { to_address : amount } - rawtx = node.createrawtransaction(inputs, outputs) - signresult = node.signrawtransaction(rawtx) - tx = CTransaction() - f = BytesIO(hex_str_to_bytes(signresult['hex'])) - tx.deserialize(f) - return tx - - def get_tests(self): - - self.coinbase_blocks = self.nodes[0].generate(2) - height = 3 # height of the next block to build - self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) + + # wait_for_verack ensures that the P2P connection is fully up. + node0.wait_for_verack() + + self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2) + self.coinbase_blocks = self.nodes[0].generate(DERSIG_HEIGHT - 2) self.nodeaddress = self.nodes[0].getnewaddress() - self.last_block_time = int(time.time()) - - ''' 298 more version 2 blocks ''' - test_blocks = [] - for i in range(298): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 2 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' Mine 749 version 3 blocks ''' - test_blocks = [] - for i in range(749): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' - Check that the new DERSIG rules are not enforced in the 750th - version 3 block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[0], self.nodeaddress, 1.0) + + self.log.info("Test that a transaction with non-DER signature can still appear in a block") + + spendtx = create_transaction(self.nodes[0], self.coinbase_blocks[0], + self.nodeaddress, 1.0) unDERify(spendtx) spendtx.rehash() - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 + tip = self.nodes[0].getbestblockhash() + block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 + block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time) + block.nVersion = 2 block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() block.rehash() block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' Mine 199 new version blocks on last valid tip ''' - test_blocks = [] - for i in range(199): - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.rehash() - block.solve() - test_blocks.append([block, True]) - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance(test_blocks, sync_every_block=False) - - ''' Mine 1 old version block ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + node0.send_and_ping(msg_block(block)) + assert_equal(self.nodes[0].getbestblockhash(), block.hash) + + self.log.info("Test that blocks must now be at least version 3") + tip = block.sha256 + block_time += 1 + block = create_block(tip, create_coinbase(DERSIG_HEIGHT), block_time) block.nVersion = 2 block.rehash() block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - ''' Mine 1 new version block ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + assert wait_until(lambda: "reject" in node0.last_message.keys()) + with mininode_lock: + assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000002)') + assert_equal(node0.last_message["reject"].data, block.sha256) + del node0.last_message["reject"] + + self.log.info("Test that transactions with non-DER signatures cannot appear in a block") block.nVersion = 3 - block.rehash() - block.solve() - self.last_block_time += 1 - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' - Check that the new DERSIG rules are enforced in the 951st version 3 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) + + spendtx = create_transaction(self.nodes[0], self.coinbase_blocks[1], + self.nodeaddress, 1.0) unDERify(spendtx) spendtx.rehash() - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 + # First we show that this tx is valid except for DERSIG by getting it + # accepted to the mempool (which we can achieve with + # -promiscuousmempoolflags). + node0.send_and_ping(msg_tx(spendtx)) + assert spendtx.hash in self.nodes[0].getrawmempool() + + # Now we verify that a block with this transaction is invalid. block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() block.rehash() block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) - ''' Mine 1 old version block, should be invalid ''' - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 2 + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) + + assert wait_until (lambda: "reject" in node0.last_message.keys()) + with mininode_lock: + # We can receive different reject messages depending on whether + # bitcoind is running with multiple script check threads. If script + # check threads are not in use, then transaction script validation + # happens sequentially, and bitcoind produces more specific reject + # reasons. + assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(node0.last_message["reject"].data, block.sha256) + if node0.last_message["reject"].code == REJECT_INVALID: + # Generic rejection when a block is invalid + assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + else: + assert b'Non-canonical DER signature' in node0.last_message["reject"].reason + + self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted") + block.vtx[1] = create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + block.hashMerkleRoot = block.calc_merkle_root() block.rehash() block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) + + node0.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) if __name__ == '__main__': BIP66Test().main() diff --git a/test/functional/bipdersig.py b/test/functional/bipdersig.py deleted file mode 100755 index 41f88fb664..0000000000 --- a/test/functional/bipdersig.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the BIP66 changeover logic.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -class BIP66Test(BitcoinTestFramework): - def __init__(self): - super().__init__() - self.num_nodes = 3 - self.setup_clean_chain = False - self.extra_args = [[], ["-blockversion=2"], ["-blockversion=3"]] - - def setup_network(self): - self.setup_nodes() - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) - self.sync_all() - - def run_test(self): - cnt = self.nodes[0].getblockcount() - - # Mine some old-version blocks - self.nodes[1].generate(100) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 100): - raise AssertionError("Failed to mine 100 version=2 blocks") - - # Mine 750 new-version blocks - for i in range(15): - self.nodes[2].generate(50) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 850): - raise AssertionError("Failed to mine 750 version=3 blocks") - - # TODO: check that new DERSIG rules are not enforced - - # Mine 1 new-version block - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 851): - raise AssertionError("Failed to mine a version=3 blocks") - - # TODO: check that new DERSIG rules are enforced - - # Mine 198 new-version blocks - for i in range(2): - self.nodes[2].generate(99) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1049): - raise AssertionError("Failed to mine 198 version=3 blocks") - - # Mine 1 old-version block - self.nodes[1].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1050): - raise AssertionError("Failed to mine a version=2 block after 949 version=3 blocks") - - # Mine 1 new-version blocks - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1051): - raise AssertionError("Failed to mine a version=3 block") - - # Mine 1 old-version blocks. This should fail - assert_raises_jsonrpc(-1, "CreateNewBlock: TestBlockValidity failed: bad-version(0x00000002)", self.nodes[1].generate, 1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1051): - raise AssertionError("Accepted a version=2 block after 950 version=3 blocks") - - # Mine 1 new-version blocks - self.nodes[2].generate(1) - self.sync_all() - if (self.nodes[0].getblockcount() != cnt + 1052): - raise AssertionError("Failed to mine a version=3 block") - -if __name__ == '__main__': - BIP66Test().main() diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index e205c6400c..0812e1b0df 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -18,16 +18,16 @@ Tests correspond to code in rpc/blockchain.cpp. """ from decimal import Decimal +import http.client import subprocess -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import (BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT) from test_framework.util import ( assert_equal, assert_raises, assert_raises_jsonrpc, assert_is_hex_string, assert_is_hash_string, - bitcoind_processes, ) @@ -139,10 +139,13 @@ class BlockchainTest(BitcoinTestFramework): self.nodes[0].generate(6) assert_equal(self.nodes[0].getblockcount(), 206) self.log.debug('Node should not stop at this height') - assert_raises(subprocess.TimeoutExpired, lambda: bitcoind_processes[0].wait(timeout=3)) - self.nodes[0].generate(1) + assert_raises(subprocess.TimeoutExpired, lambda: self.nodes[0].process.wait(timeout=3)) + try: + self.nodes[0].generate(1) + except (ConnectionError, http.client.BadStatusLine): + pass # The node already shut down before response self.log.debug('Node should stop at this height...') - bitcoind_processes[0].wait(timeout=3) + self.nodes[0].process.wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) self.nodes[0] = self.start_node(0, self.options.tmpdir) assert_equal(self.nodes[0].getblockcount(), 207) diff --git a/test/functional/bumpfee.py b/test/functional/bumpfee.py index 569db7ced5..9633ffdebb 100755 --- a/test/functional/bumpfee.py +++ b/test/functional/bumpfee.py @@ -41,8 +41,7 @@ class BumpFeeTest(BitcoinTestFramework): self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) # Encrypt wallet for test_locked_wallet_fails test - self.nodes[1].encryptwallet(WALLET_PASSPHRASE) - bitcoind_processes[1].wait() + self.nodes[1].node_encrypt_wallet(WALLET_PASSPHRASE) self.nodes[1] = self.start_node(1, self.options.tmpdir, extra_args[1]) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) @@ -90,7 +89,7 @@ def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 - # check that bumped_tx propogates, original tx was evicted and has a wallet conflict + # check that bumped_tx propagates, original tx was evicted and has a wallet conflict sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert bumped_tx["txid"] in peer_node.getrawmempool() diff --git a/test/functional/dbcrash.py b/test/functional/dbcrash.py new file mode 100755 index 0000000000..8339305f5e --- /dev/null +++ b/test/functional/dbcrash.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test recovery from a crash during chainstate writing. + +- 4 nodes + * node0, node1, and node2 will have different dbcrash ratios, and different + dbcache sizes + * node3 will be a regular node, with no crashing. + * The nodes will not connect to each other. + +- use default test framework starting chain. initialize starting_tip_height to + tip height. + +- Main loop: + * generate lots of transactions on node3, enough to fill up a block. + * uniformly randomly pick a tip height from starting_tip_height to + tip_height; with probability 1/(height_difference+4), invalidate this block. + * mine enough blocks to overtake tip_height at start of loop. + * for each node in [node0,node1,node2]: + - for each mined block: + * submit block to node + * if node crashed on/after submitting: + - restart until recovery succeeds + - check that utxo matches node3 using gettxoutsetinfo""" + +import errno +import http.client +import random +import sys +import time + +from test_framework.mininode import * +from test_framework.script import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest] +try: + HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected) +except AttributeError: + pass + +class ChainstateWriteCrashTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + + # Set -maxmempool=0 to turn off mempool memory sharing with dbcache + # Set -rpcservertimeout=900 to reduce socket disconnects in this + # long-running test + self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000"] + + # Set different crash ratios and cache sizes. Note that not all of + # -dbcache goes to pcoinsTip. + self.node0_args = ["-dbcrashratio=8", "-dbcache=4"] + self.base_args + self.node1_args = ["-dbcrashratio=16", "-dbcache=8"] + self.base_args + self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args + + # Node3 is a normal node with default args, except will mine full blocks + self.node3_args = ["-blockmaxweight=4000000"] + self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] + + def setup_network(self): + self.setup_nodes() + # Leave them unconnected, we'll use submitblock directly in this test + + def restart_node(self, node_index, expected_tip): + """Start up a given node id, wait for the tip to reach the given block hash, and calculate the utxo hash. + + Exceptions on startup should indicate node crash (due to -dbcrashratio), in which case we try again. Give up + after 60 seconds. Returns the utxo hash of the given node.""" + + time_start = time.time() + while time.time() - time_start < 60: + try: + # Any of these RPC calls could throw due to node crash + self.nodes[node_index] = self.start_node(node_index, self.options.tmpdir, self.extra_args[node_index]) + self.nodes[node_index].waitforblock(expected_tip) + utxo_hash = self.nodes[node_index].gettxoutsetinfo()['hash_serialized_2'] + return utxo_hash + except: + # An exception here should mean the node is about to crash. + # If bitcoind exits, then try again. wait_for_node_exit() + # should raise an exception if bitcoind doesn't exit. + self.wait_for_node_exit(node_index, timeout=10) + self.crashed_on_restart += 1 + time.sleep(1) + + # If we got here, bitcoind isn't coming back up on restart. Could be a + # bug in bitcoind, or we've gotten unlucky with our dbcrash ratio -- + # perhaps we generated a test case that blew up our cache? + # TODO: If this happens a lot, we should try to restart without -dbcrashratio + # and make sure that recovery happens. + raise AssertionError("Unable to successfully restart node %d in allotted time", node_index) + + def submit_block_catch_error(self, node_index, block): + """Try submitting a block to the given node. + + Catch any exceptions that indicate the node has crashed. + Returns true if the block was submitted successfully; false otherwise.""" + + try: + self.nodes[node_index].submitblock(block) + return True + except http.client.BadStatusLine as e: + # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error. + if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''": + self.log.debug("node %d submitblock raised exception: %s", node_index, e) + return False + else: + raise + except tuple(HTTP_DISCONNECT_ERRORS) as e: + self.log.debug("node %d submitblock raised exception: %s", node_index, e) + return False + except OSError as e: + self.log.debug("node %d submitblock raised OSError exception: errno=%s", node_index, e.errno) + if e.errno in [errno.EPIPE, errno.ECONNREFUSED, errno.ECONNRESET]: + # The node has likely crashed + return False + else: + # Unexpected exception, raise + raise + + def sync_node3blocks(self, block_hashes): + """Use submitblock to sync node3's chain with the other nodes + + If submitblock fails, restart the node and get the new utxo hash. + If any nodes crash while updating, we'll compare utxo hashes to + ensure recovery was successful.""" + + node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2'] + + # Retrieve all the blocks from node3 + blocks = [] + for block_hash in block_hashes: + blocks.append([block_hash, self.nodes[3].getblock(block_hash, 0)]) + + # Deliver each block to each other node + for i in range(3): + nodei_utxo_hash = None + self.log.debug("Syncing blocks to node %d", i) + for (block_hash, block) in blocks: + # Get the block from node3, and submit to node_i + self.log.debug("submitting block %s", block_hash) + if not self.submit_block_catch_error(i, block): + # TODO: more carefully check that the crash is due to -dbcrashratio + # (change the exit code perhaps, and check that here?) + self.wait_for_node_exit(i, timeout=30) + self.log.debug("Restarting node %d after block hash %s", i, block_hash) + nodei_utxo_hash = self.restart_node(i, block_hash) + assert nodei_utxo_hash is not None + self.restart_counts[i] += 1 + else: + # Clear it out after successful submitblock calls -- the cached + # utxo hash will no longer be correct + nodei_utxo_hash = None + + # Check that the utxo hash matches node3's utxo set + # NOTE: we only check the utxo set if we had to restart the node + # after the last block submitted: + # - checking the utxo hash causes a cache flush, which we don't + # want to do every time; so + # - we only update the utxo cache after a node restart, since flushing + # the cache is a no-op at that point + if nodei_utxo_hash is not None: + self.log.debug("Checking txoutsetinfo matches for node %d", i) + assert_equal(nodei_utxo_hash, node3_utxo_hash) + + def verify_utxo_hash(self): + """Verify that the utxo hash of each node matches node3. + + Restart any nodes that crash while querying.""" + node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2'] + self.log.info("Verifying utxo hash matches for all nodes") + + for i in range(3): + try: + nodei_utxo_hash = self.nodes[i].gettxoutsetinfo()['hash_serialized_2'] + except OSError: + # probably a crash on db flushing + nodei_utxo_hash = self.restart_node(i, self.nodes[3].getbestblockhash()) + assert_equal(nodei_utxo_hash, node3_utxo_hash) + + def generate_small_transactions(self, node, count, utxo_list): + FEE = 1000 # TODO: replace this with node relay fee based calculation + num_transactions = 0 + random.shuffle(utxo_list) + while len(utxo_list) >= 2 and num_transactions < count: + tx = CTransaction() + input_amount = 0 + for i in range(2): + utxo = utxo_list.pop() + tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout']))) + input_amount += int(utxo['amount'] * COIN) + output_amount = (input_amount - FEE) // 3 + + if output_amount <= 0: + # Sanity check -- if we chose inputs that are too small, skip + continue + + for i in range(3): + tx.vout.append(CTxOut(output_amount, hex_str_to_bytes(utxo['scriptPubKey']))) + + # Sign and send the transaction to get into the mempool + tx_signed_hex = node.signrawtransaction(ToHex(tx))['hex'] + node.sendrawtransaction(tx_signed_hex) + num_transactions += 1 + + def run_test(self): + # Track test coverage statistics + self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2 + self.crashed_on_restart = 0 # Track count of crashes during recovery + + # Start by creating a lot of utxos on node3 + initial_height = self.nodes[3].getblockcount() + utxo_list = create_confirmed_utxos(self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000) + self.log.info("Prepped %d utxo entries", len(utxo_list)) + + # Sync these blocks with the other nodes + block_hashes_to_sync = [] + for height in range(initial_height + 1, self.nodes[3].getblockcount() + 1): + block_hashes_to_sync.append(self.nodes[3].getblockhash(height)) + + self.log.debug("Syncing %d blocks with other nodes", len(block_hashes_to_sync)) + # Syncing the blocks could cause nodes to crash, so the test begins here. + self.sync_node3blocks(block_hashes_to_sync) + + starting_tip_height = self.nodes[3].getblockcount() + + # Main test loop: + # each time through the loop, generate a bunch of transactions, + # and then either mine a single new block on the tip, or some-sized reorg. + for i in range(40): + self.log.info("Iteration %d, generating 2500 transactions %s", i, self.restart_counts) + # Generate a bunch of small-ish transactions + self.generate_small_transactions(self.nodes[3], 2500, utxo_list) + # Pick a random block between current tip, and starting tip + current_height = self.nodes[3].getblockcount() + random_height = random.randint(starting_tip_height, current_height) + self.log.debug("At height %d, considering height %d", current_height, random_height) + if random_height > starting_tip_height: + # Randomly reorg from this point with some probability (1/4 for + # tip, 1/5 for tip-1, ...) + if random.random() < 1.0 / (current_height + 4 - random_height): + self.log.debug("Invalidating block at height %d", random_height) + self.nodes[3].invalidateblock(self.nodes[3].getblockhash(random_height)) + + # Now generate new blocks until we pass the old tip height + self.log.debug("Mining longer tip") + block_hashes = [] + while current_height + 1 > self.nodes[3].getblockcount(): + block_hashes.extend(self.nodes[3].generate(min(10, current_height + 1 - self.nodes[3].getblockcount()))) + self.log.debug("Syncing %d new blocks...", len(block_hashes)) + self.sync_node3blocks(block_hashes) + utxo_list = self.nodes[3].listunspent() + self.log.debug("Node3 utxo count: %d", len(utxo_list)) + + # Check that the utxo hashes agree with node3 + # Useful side effect: each utxo cache gets flushed here, so that we + # won't get crashes on shutdown at the end of the test. + self.verify_utxo_hash() + + # Check the test coverage + self.log.info("Restarted nodes: %s; crashes on restart: %d", self.restart_counts, self.crashed_on_restart) + + # If no nodes were restarted, we didn't test anything. + assert self.restart_counts != [0, 0, 0] + + # Make sure we tested the case of crash-during-recovery. + assert self.crashed_on_restart > 0 + + # Warn if any of the nodes escaped restart. + for i in range(3): + if self.restart_counts[i] == 0: + self.log.warn("Node %d never crashed during utxo flush!", i) + +if __name__ == "__main__": + ChainstateWriteCrashTest().main() diff --git a/test/functional/example_test.py b/test/functional/example_test.py new file mode 100755 index 0000000000..79940a4264 --- /dev/null +++ b/test/functional/example_test.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""An example functional test + +The module-level docstring should include a high-level description of +what the test is doing. It's the first thing people see when they open +the file and should give the reader information about *what* the test +is testing and *how* it's being tested +""" +# Imports should be in PEP8 ordering (std library first, then third party +# libraries then local imports). +from collections import defaultdict + +# Avoid wildcard * imports if possible +from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.mininode import ( + CInv, + NetworkThread, + NodeConn, + NodeConnCB, + mininode_lock, + msg_block, + msg_getdata, + wait_until, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + p2p_port, +) + +# NodeConnCB is a class containing callbacks to be executed when a P2P +# message is received from the node-under-test. Subclass NodeConnCB and +# override the on_*() methods if you need custom behaviour. +class BaseNode(NodeConnCB): + def __init__(self): + """Initialize the NodeConnCB + + Used to inialize custom properties for the Node that aren't + included by default in the base class. Be aware that the NodeConnCB + base class already stores a counter for each P2P message type and the + last received message of each type, which should be sufficient for the + needs of most tests. + + Call super().__init__() first for standard initialization and then + initialize custom properties.""" + super().__init__() + # Stores a dictionary of all blocks received + self.block_receive_map = defaultdict(int) + + def on_block(self, conn, message): + """Override the standard on_block callback + + Store the hash of a received block in the dictionary.""" + message.block.calc_sha256() + self.block_receive_map[message.block.sha256] += 1 + + def on_inv(self, conn, message): + """Override the standard on_inv callback""" + pass + +def custom_function(): + """Do some custom behaviour + + If this function is more generally useful for other tests, consider + moving it to a module in test_framework.""" + # self.log.info("running custom_function") # Oops! Can't run self.log outside the BitcoinTestFramework + pass + +class ExampleTest(BitcoinTestFramework): + # Each functional test is a subclass of the BitcoinTestFramework class. + + # Override the __init__(), add_options(), setup_chain(), setup_network() + # and setup_nodes() methods to customize the test setup as required. + + def __init__(self): + """Initialize the test + + Call super().__init__() first, and then override any test parameters + for your individual test.""" + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + # Use self.extra_args to change command-line arguments for the nodes + self.extra_args = [[], ["-logips"], []] + + # self.log.info("I've finished __init__") # Oops! Can't run self.log before run_test() + + # Use add_options() to add specific command-line options for your test. + # In practice this is not used very much, since the tests are mostly written + # to be run in automated environments without command-line options. + # def add_options() + # pass + + # Use setup_chain() to customize the node data directories. In practice + # this is not used very much since the default behaviour is almost always + # fine + # def setup_chain(): + # pass + + def setup_network(self): + """Setup the test network topology + + Often you won't need to override this, since the standard network topology + (linear: node0 <-> node1 <-> node2 <-> ...) is fine for most tests. + + If you do override this method, remember to start the nodes, assign + them to self.nodes, connect them and then sync.""" + + self.setup_nodes() + + # In this test, we're not connecting node2 to node0 or node1. Calls to + # sync_all() should not include node2, since we're not expecting it to + # sync. + connect_nodes(self.nodes[0], 1) + self.sync_all([self.nodes[0:1]]) + + # Use setup_nodes() to customize the node start behaviour (for example if + # you don't want to start all nodes at the start of the test). + # def setup_nodes(): + # pass + + def custom_method(self): + """Do some custom behaviour for this test + + Define it in a method here because you're going to use it repeatedly. + If you think it's useful in general, consider moving it to the base + BitcoinTestFramework class so other tests can use it.""" + + self.log.info("Running custom_method") + + def run_test(self): + """Main test logic""" + + # Create a P2P connection to one of the nodes + node0 = BaseNode() + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) + node0.add_connection(connections[0]) + + # Start up network handling in another thread. This needs to be called + # after the P2P connections have been created. + NetworkThread().start() + # wait_for_verack ensures that the P2P connection is fully up. + node0.wait_for_verack() + + # Generating a block on one of the nodes will get us out of IBD + blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] + self.sync_all([self.nodes[0:1]]) + + # Notice above how we called an RPC by calling a method with the same + # name on the node object. Notice also how we used a keyword argument + # to specify a named RPC argument. Neither of those are defined on the + # node object. Instead there's some __getattr__() magic going on under + # the covers to dispatch unrecognised attribute calls to the RPC + # interface. + + # Logs are nice. Do plenty of them. They can be used in place of comments for + # breaking the test into sub-sections. + self.log.info("Starting test!") + + self.log.info("Calling a custom function") + custom_function() + + self.log.info("Calling a custom method") + self.custom_method() + + self.log.info("Create some blocks") + self.tip = int(self.nodes[0].getbestblockhash(), 16) + self.block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 + + height = 1 + + for i in range(10): + # Use the mininode and blocktools functionality to manually build a block + # Calling the generate() rpc is easier, but this allows us to exactly + # control the blocks and transactions. + block = create_block(self.tip, create_coinbase(height), self.block_time) + block.solve() + block_message = msg_block(block) + # Send message is used to send a P2P message to the node over our NodeConn connection + node0.send_message(block_message) + self.tip = block.sha256 + blocks.append(self.tip) + self.block_time += 1 + height += 1 + + self.log.info("Wait for node1 to reach current tip (height 11) using RPC") + self.nodes[1].waitforblockheight(11) + + self.log.info("Connect node2 and node1") + connect_nodes(self.nodes[1], 2) + + self.log.info("Add P2P connection to node2") + node2 = BaseNode() + connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) + node2.add_connection(connections[1]) + node2.wait_for_verack() + + self.log.info("Wait for node2 reach current tip. Test that it has propagated all the blocks to us") + + getdata_request = msg_getdata() + for block in blocks: + getdata_request.inv.append(CInv(2, block)) + node2.send_message(getdata_request) + + # wait_until() will loop until a predicate condition is met. Use it to test properties of the + # NodeConnCB objects. + assert wait_until(lambda: sorted(blocks) == sorted(list(node2.block_receive_map.keys())), timeout=5) + + self.log.info("Check that each block was received only once") + # The network thread uses a global lock on data access to the NodeConn objects when sending and receiving + # messages. The test thread should acquire the global lock before accessing any NodeConn data to avoid locking + # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. + with mininode_lock: + for block in node2.block_receive_map.values(): + assert_equal(block, 1) + +if __name__ == '__main__': + ExampleTest().main() diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index 0a3166b89b..f2f4efcf28 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT from test_framework.util import * @@ -451,8 +451,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.stop_node(0) self.stop_node(2) self.stop_node(3) - self.nodes[1].encryptwallet("test") - bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) + self.nodes[1].node_encrypt_wallet("test") self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like @@ -636,20 +635,9 @@ class RawTransactionsTest(BitcoinTestFramework): assert_fee_amount(result2['fee'], count_bytes(result2['hex']), 2 * result_fee_rate) assert_fee_amount(result3['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) - ############################# - # Test address reuse option # - ############################# - - result3 = self.nodes[3].fundrawtransaction(rawtx, {"reserveChangeKey": False}) - res_dec = self.nodes[0].decoderawtransaction(result3["hex"]) - changeaddress = "" - for out in res_dec['vout']: - if out['value'] > 1.0: - changeaddress += out['scriptPubKey']['addresses'][0] - assert(changeaddress != "") - nextaddr = self.nodes[3].getrawchangeaddress() - # frt should not have removed the key from the keypool - assert(changeaddress == nextaddr) + ################################ + # Test no address reuse occurs # + ################################ result3 = self.nodes[3].fundrawtransaction(rawtx) res_dec = self.nodes[0].decoderawtransaction(result3["hex"]) diff --git a/test/functional/getblocktemplate_longpoll.py b/test/functional/getblocktemplate_longpoll.py index bbe1dda5f7..cca30e2688 100755 --- a/test/functional/getblocktemplate_longpoll.py +++ b/test/functional/getblocktemplate_longpoll.py @@ -17,7 +17,7 @@ class LongpollThread(threading.Thread): self.longpollid = templat['longpollid'] # create a new connection to the node, we can't use the same # connection from two threads - self.node = get_rpc_proxy(node.url, 1, timeout=600) + self.node = get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) def run(self): self.node.getblocktemplate({'longpollid':self.longpollid}) diff --git a/test/functional/getblocktemplate_proposals.py b/test/functional/getblocktemplate_proposals.py deleted file mode 100755 index fca99c7df5..0000000000 --- a/test/functional/getblocktemplate_proposals.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test block proposals with getblocktemplate.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -from binascii import a2b_hex, b2a_hex -from hashlib import sha256 -from struct import pack - -def b2x(b): - return b2a_hex(b).decode('ascii') - -# NOTE: This does not work for signed numbers (set the high bit) or zero (use b'\0') -def encodeUNum(n): - s = bytearray(b'\1') - while n > 127: - s[0] += 1 - s.append(n % 256) - n //= 256 - s.append(n) - return bytes(s) - -def varlenEncode(n): - if n < 0xfd: - return pack('<B', n) - if n <= 0xffff: - return b'\xfd' + pack('<H', n) - if n <= 0xffffffff: - return b'\xfe' + pack('<L', n) - return b'\xff' + pack('<Q', n) - -def dblsha(b): - return sha256(sha256(b).digest()).digest() - -def genmrklroot(leaflist): - cur = leaflist - while len(cur) > 1: - n = [] - if len(cur) & 1: - cur.append(cur[-1]) - for i in range(0, len(cur), 2): - n.append(dblsha(cur[i] + cur[i+1])) - cur = n - return cur[0] - -def template_to_bytearray(tmpl, txlist): - blkver = pack('<L', tmpl['version']) - mrklroot = genmrklroot(list(dblsha(a) for a in txlist)) - timestamp = pack('<L', tmpl['curtime']) - nonce = b'\0\0\0\0' - blk = blkver + a2b_hex(tmpl['previousblockhash'])[::-1] + mrklroot + timestamp + a2b_hex(tmpl['bits'])[::-1] + nonce - blk += varlenEncode(len(txlist)) - for tx in txlist: - blk += tx - return bytearray(blk) - -def template_to_hex(tmpl, txlist): - return b2x(template_to_bytearray(tmpl, txlist)) - -def assert_template(node, tmpl, txlist, expect): - rsp = node.getblocktemplate({'data':template_to_hex(tmpl, txlist),'mode':'proposal'}) - if rsp != expect: - raise AssertionError('unexpected: %s' % (rsp,)) - -class GetBlockTemplateProposalTest(BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 2 - self.setup_clean_chain = False - - def run_test(self): - node = self.nodes[0] - node.generate(1) # Mine a block to leave initial block download - tmpl = node.getblocktemplate() - if 'coinbasetxn' not in tmpl: - rawcoinbase = encodeUNum(tmpl['height']) - rawcoinbase += b'\x01-' - hexcoinbase = b2x(rawcoinbase) - hexoutval = b2x(pack('<Q', tmpl['coinbasevalue'])) - tmpl['coinbasetxn'] = {'data': '01000000' + '01' + '0000000000000000000000000000000000000000000000000000000000000000ffffffff' + ('%02x' % (len(rawcoinbase),)) + hexcoinbase + 'fffffffe' + '01' + hexoutval + '00' + '00000000'} - txlist = list(bytearray(a2b_hex(a['data'])) for a in (tmpl['coinbasetxn'],) + tuple(tmpl['transactions'])) - - # Test 0: Capability advertised - assert('proposal' in tmpl['capabilities']) - - # NOTE: This test currently FAILS (regtest mode doesn't enforce block height in coinbase) - ## Test 1: Bad height in coinbase - #txlist[0][4+1+36+1+1] += 1 - #assert_template(node, tmpl, txlist, 'FIXME') - #txlist[0][4+1+36+1+1] -= 1 - - # Test 2: Bad input hash for gen tx - txlist[0][4+1] += 1 - assert_template(node, tmpl, txlist, 'bad-cb-missing') - txlist[0][4+1] -= 1 - - # Test 3: Truncated final tx - lastbyte = txlist[-1].pop() - assert_raises_jsonrpc(-22, "Block decode failed", assert_template, node, tmpl, txlist, 'n/a') - txlist[-1].append(lastbyte) - - # Test 4: Add an invalid tx to the end (duplicate of gen tx) - txlist.append(txlist[0]) - assert_template(node, tmpl, txlist, 'bad-txns-duplicate') - txlist.pop() - - # Test 5: Add an invalid tx to the end (non-duplicate) - txlist.append(bytearray(txlist[0])) - txlist[-1][4+1] = 0xff - assert_template(node, tmpl, txlist, 'bad-txns-inputs-missingorspent') - txlist.pop() - - # Test 6: Future tx lock time - txlist[0][-4:] = b'\xff\xff\xff\xff' - assert_template(node, tmpl, txlist, 'bad-txns-nonfinal') - txlist[0][-4:] = b'\0\0\0\0' - - # Test 7: Bad tx count - txlist.append(b'') - assert_raises_jsonrpc(-22, 'Block decode failed', assert_template, node, tmpl, txlist, 'n/a') - txlist.pop() - - # Test 8: Bad bits - realbits = tmpl['bits'] - tmpl['bits'] = '1c0000ff' # impossible in the real world - assert_template(node, tmpl, txlist, 'bad-diffbits') - tmpl['bits'] = realbits - - # Test 9: Bad merkle root - rawtmpl = template_to_bytearray(tmpl, txlist) - rawtmpl[4+32] = (rawtmpl[4+32] + 1) % 0x100 - rsp = node.getblocktemplate({'data':b2x(rawtmpl),'mode':'proposal'}) - if rsp != 'bad-txnmrklroot': - raise AssertionError('unexpected: %s' % (rsp,)) - - # Test 10: Bad timestamps - realtime = tmpl['curtime'] - tmpl['curtime'] = 0x7fffffff - assert_template(node, tmpl, txlist, 'time-too-new') - tmpl['curtime'] = 0 - assert_template(node, tmpl, txlist, 'time-too-old') - tmpl['curtime'] = realtime - - # Test 11: Valid block - assert_template(node, tmpl, txlist, None) - - # Test 12: Orphan block - tmpl['previousblockhash'] = 'ff00' * 16 - assert_template(node, tmpl, txlist, 'inconclusive-not-best-prevblk') - -if __name__ == '__main__': - GetBlockTemplateProposalTest().main() diff --git a/test/functional/keypool-topup.py b/test/functional/keypool-topup.py new file mode 100755 index 0000000000..0e0c0ea74b --- /dev/null +++ b/test/functional/keypool-topup.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test HD Wallet keypool restore function. + +Two nodes. Node1 is under test. Node0 is providing transactions and generating blocks. + +- Start node1, shutdown and backup wallet. +- Generate 110 keys (enough to drain the keypool). Store key 90 (in the initial keypool) and key 110 (beyond the initial keypool). Send funds to key 90 and key 110. +- Stop node1, clear the datadir, move wallet file back into the datadir and restart node1. +- connect node1 to node0. Verify that they sync and node1 receives its funds.""" +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes_bi, + sync_blocks, +) + +class KeypoolRestoreTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [['-usehd=0'], ['-usehd=1', '-keypool=100', '-keypoolmin=20']] + + def run_test(self): + self.tmpdir = self.options.tmpdir + self.nodes[0].generate(101) + + self.log.info("Make backup of wallet") + + self.stop_node(1) + + shutil.copyfile(self.tmpdir + "/node1/regtest/wallet.dat", self.tmpdir + "/wallet.bak") + self.nodes[1] = self.start_node(1, self.tmpdir, self.extra_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + + self.log.info("Generate keys for wallet") + + for _ in range(90): + addr_oldpool = self.nodes[1].getnewaddress() + for _ in range(20): + addr_extpool = self.nodes[1].getnewaddress() + + self.log.info("Send funds to wallet") + + self.nodes[0].sendtoaddress(addr_oldpool, 10) + self.nodes[0].generate(1) + self.nodes[0].sendtoaddress(addr_extpool, 5) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + self.log.info("Restart node with wallet backup") + + self.stop_node(1) + + shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallet.dat") + + self.log.info("Verify keypool is restored and balance is correct") + + self.nodes[1] = self.start_node(1, self.tmpdir, self.extra_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + self.sync_all() + + assert_equal(self.nodes[1].getbalance(), 15) + assert_equal(self.nodes[1].listtransactions()[0]['category'], "receive") + + # Check that we have marked all keys up to the used keypool key as used + assert_equal(self.nodes[1].validateaddress(self.nodes[1].getnewaddress())['hdkeypath'], "m/0'/0'/111'") + +if __name__ == '__main__': + KeypoolRestoreTest().main() diff --git a/test/functional/keypool.py b/test/functional/keypool.py index f23a427d1f..3e7bb0ee07 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -17,8 +17,7 @@ class KeyPoolTest(BitcoinTestFramework): assert(addr_before_encrypting_data['hdmasterkeyid'] == wallet_info_old['hdmasterkeyid']) # Encrypt wallet and wait to terminate - nodes[0].encryptwallet('test') - bitcoind_processes[0].wait() + nodes[0].node_encrypt_wallet('test') # Restart node 0 nodes[0] = self.start_node(0, self.options.tmpdir) # Keep creating keys diff --git a/test/functional/listsinceblock.py b/test/functional/listsinceblock.py index f3d41e573e..ce2d556ef0 100755 --- a/test/functional/listsinceblock.py +++ b/test/functional/listsinceblock.py @@ -14,7 +14,15 @@ class ListSinceBlockTest (BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 4 - def run_test (self): + def run_test(self): + self.nodes[2].generate(101) + self.sync_all() + + self.test_reorg() + self.test_double_spend() + self.test_double_send() + + def test_reorg(self): ''' `listsinceblock` did not behave correctly when handed a block that was no longer in the main chain: @@ -43,14 +51,6 @@ class ListSinceBlockTest (BitcoinTestFramework): This test only checks that [tx0] is present. ''' - self.nodes[2].generate(101) - self.sync_all() - - assert_equal(self.nodes[0].getbalance(), 0) - assert_equal(self.nodes[1].getbalance(), 0) - assert_equal(self.nodes[2].getbalance(), 50) - assert_equal(self.nodes[3].getbalance(), 0) - # Split network into two self.split_network() @@ -73,7 +73,177 @@ class ListSinceBlockTest (BitcoinTestFramework): if tx['txid'] == senttx: found = True break - assert_equal(found, True) + assert found + + def test_double_spend(self): + ''' + This tests the case where the same UTXO is spent twice on two separate + blocks as part of a reorg. + + ab0 + / \ + aa1 [tx1] bb1 [tx2] + | | + aa2 bb2 + | | + aa3 bb3 + | + bb4 + + Problematic case: + + 1. User 1 receives BTC in tx1 from utxo1 in block aa1. + 2. User 2 receives BTC in tx2 from utxo1 (same) in block bb1 + 3. User 1 sees 2 confirmations at block aa3. + 4. Reorg into bb chain. + 5. User 1 asks `listsinceblock aa3` and does not see that tx1 is now + invalidated. + + Currently the solution to this is to detect that a reorg'd block is + asked for in listsinceblock, and to iterate back over existing blocks up + until the fork point, and to include all transactions that relate to the + node wallet. + ''' + + self.sync_all() + + # Split network into two + self.split_network() + + # share utxo between nodes[1] and nodes[2] + utxos = self.nodes[2].listunspent() + utxo = utxos[0] + privkey = self.nodes[2].dumpprivkey(utxo['address']) + self.nodes[1].importprivkey(privkey) + + # send from nodes[1] using utxo to nodes[0] + change = '%.8f' % (float(utxo['amount']) - 1.0003) + recipientDict = { + self.nodes[0].getnewaddress(): 1, + self.nodes[1].getnewaddress(): change, + } + utxoDicts = [{ + 'txid': utxo['txid'], + 'vout': utxo['vout'], + }] + txid1 = self.nodes[1].sendrawtransaction( + self.nodes[1].signrawtransaction( + self.nodes[1].createrawtransaction(utxoDicts, recipientDict))['hex']) + + # send from nodes[2] using utxo to nodes[3] + recipientDict2 = { + self.nodes[3].getnewaddress(): 1, + self.nodes[2].getnewaddress(): change, + } + self.nodes[2].sendrawtransaction( + self.nodes[2].signrawtransaction( + self.nodes[2].createrawtransaction(utxoDicts, recipientDict2))['hex']) + + # generate on both sides + lastblockhash = self.nodes[1].generate(3)[2] + self.nodes[2].generate(4) + + self.join_network() + + self.sync_all() + + # gettransaction should work for txid1 + assert self.nodes[0].gettransaction(txid1)['txid'] == txid1, "gettransaction failed to find txid1" + + # listsinceblock(lastblockhash) should now include txid1, as seen from nodes[0] + lsbres = self.nodes[0].listsinceblock(lastblockhash) + assert any(tx['txid'] == txid1 for tx in lsbres['removed']) + + # but it should not include 'removed' if include_removed=false + lsbres2 = self.nodes[0].listsinceblock(blockhash=lastblockhash, include_removed=False) + assert 'removed' not in lsbres2 + + def test_double_send(self): + ''' + This tests the case where the same transaction is submitted twice on two + separate blocks as part of a reorg. The former will vanish and the + latter will appear as the true transaction (with confirmations dropping + as a result). + + ab0 + / \ + aa1 [tx1] bb1 + | | + aa2 bb2 + | | + aa3 bb3 [tx1] + | + bb4 + + Asserted: + + 1. tx1 is listed in listsinceblock. + 2. It is included in 'removed' as it was removed, even though it is now + present in a different block. + 3. It is listed with a confirmations count of 2 (bb3, bb4), not + 3 (aa1, aa2, aa3). + ''' + + self.sync_all() + + # Split network into two + self.split_network() + + # create and sign a transaction + utxos = self.nodes[2].listunspent() + utxo = utxos[0] + change = '%.8f' % (float(utxo['amount']) - 1.0003) + recipientDict = { + self.nodes[0].getnewaddress(): 1, + self.nodes[2].getnewaddress(): change, + } + utxoDicts = [{ + 'txid': utxo['txid'], + 'vout': utxo['vout'], + }] + signedtxres = self.nodes[2].signrawtransaction( + self.nodes[2].createrawtransaction(utxoDicts, recipientDict)) + assert signedtxres['complete'] + + signedtx = signedtxres['hex'] + + # send from nodes[1]; this will end up in aa1 + txid1 = self.nodes[1].sendrawtransaction(signedtx) + + # generate bb1-bb2 on right side + self.nodes[2].generate(2) + + # send from nodes[2]; this will end up in bb3 + txid2 = self.nodes[2].sendrawtransaction(signedtx) + + assert_equal(txid1, txid2) + + # generate on both sides + lastblockhash = self.nodes[1].generate(3)[2] + self.nodes[2].generate(2) + + self.join_network() + + self.sync_all() + + # gettransaction should work for txid1 + self.nodes[0].gettransaction(txid1) + + # listsinceblock(lastblockhash) should now include txid1 in transactions + # as well as in removed + lsbres = self.nodes[0].listsinceblock(lastblockhash) + assert any(tx['txid'] == txid1 for tx in lsbres['transactions']) + assert any(tx['txid'] == txid1 for tx in lsbres['removed']) + + # find transaction and ensure confirmations is valid + for tx in lsbres['transactions']: + if tx['txid'] == txid1: + assert_equal(tx['confirmations'], 2) + + # the same check for the removed array; confirmations should STILL be 2 + for tx in lsbres['removed']: + if tx['txid'] == txid1: + assert_equal(tx['confirmations'], 2) if __name__ == '__main__': ListSinceBlockTest().main() diff --git a/test/functional/listtransactions.py b/test/functional/listtransactions.py index f69f1c5724..f75a8e29cc 100755 --- a/test/functional/listtransactions.py +++ b/test/functional/listtransactions.py @@ -23,7 +23,7 @@ class ListTransactionsTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime - enable_mocktime() + self.enable_mocktime() self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): diff --git a/test/functional/mining.py b/test/functional/mining.py new file mode 100755 index 0000000000..dbd4e29eca --- /dev/null +++ b/test/functional/mining.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mining RPCs + +- getblocktemplate proposal mode +- submitblock""" + +from binascii import b2a_hex +import copy + +from test_framework.blocktools import create_coinbase +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import CBlock +from test_framework.util import * + +def b2x(b): + return b2a_hex(b).decode('ascii') + +def assert_template(node, block, expect, rehash=True): + if rehash: + block.hashMerkleRoot = block.calc_merkle_root() + rsp = node.getblocktemplate({'data': b2x(block.serialize()), 'mode': 'proposal'}) + assert_equal(rsp, expect) + +class MiningTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + + def run_test(self): + node = self.nodes[0] + # Mine a block to leave initial block download + node.generate(1) + tmpl = node.getblocktemplate() + self.log.info("getblocktemplate: Test capability advertised") + assert 'proposal' in tmpl['capabilities'] + assert 'coinbasetxn' not in tmpl + + coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1) + # sequence numbers must not be max for nLockTime to have effect + coinbase_tx.vin[0].nSequence = 2 ** 32 - 2 + coinbase_tx.rehash() + + block = CBlock() + block.nVersion = tmpl["version"] + block.hashPrevBlock = int(tmpl["previousblockhash"], 16) + block.nTime = tmpl["curtime"] + block.nBits = int(tmpl["bits"], 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + + self.log.info("getblocktemplate: Test valid block") + assert_template(node, block, None) + + self.log.info("submitblock: Test block decode failure") + assert_raises_jsonrpc(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + + self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx[0].vin[0].prevout.hash += 1 + bad_block.vtx[0].rehash() + assert_template(node, bad_block, 'bad-cb-missing') + + self.log.info("submitblock: Test invalid coinbase transaction") + assert_raises_jsonrpc(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + + self.log.info("getblocktemplate: Test truncated final transaction") + assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'}) + + self.log.info("getblocktemplate: Test duplicate transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx.append(bad_block.vtx[0]) + assert_template(node, bad_block, 'bad-txns-duplicate') + + self.log.info("getblocktemplate: Test invalid transaction") + bad_block = copy.deepcopy(block) + bad_tx = copy.deepcopy(bad_block.vtx[0]) + bad_tx.vin[0].prevout.hash = 255 + bad_tx.rehash() + bad_block.vtx.append(bad_tx) + assert_template(node, bad_block, 'bad-txns-inputs-missingorspent') + + self.log.info("getblocktemplate: Test nonfinal transaction") + bad_block = copy.deepcopy(block) + bad_block.vtx[0].nLockTime = 2 ** 32 - 1 + bad_block.vtx[0].rehash() + assert_template(node, bad_block, 'bad-txns-nonfinal') + + self.log.info("getblocktemplate: Test bad tx count") + # The tx count is immediately after the block header + TX_COUNT_OFFSET = 80 + bad_block_sn = bytearray(block.serialize()) + assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) + bad_block_sn[TX_COUNT_OFFSET] += 1 + assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'}) + + self.log.info("getblocktemplate: Test bad bits") + bad_block = copy.deepcopy(block) + bad_block.nBits = 469762303 # impossible in the real world + assert_template(node, bad_block, 'bad-diffbits') + + self.log.info("getblocktemplate: Test bad merkle root") + bad_block = copy.deepcopy(block) + bad_block.hashMerkleRoot += 1 + assert_template(node, bad_block, 'bad-txnmrklroot', False) + + self.log.info("getblocktemplate: Test bad timestamps") + bad_block = copy.deepcopy(block) + bad_block.nTime = 2 ** 31 - 1 + assert_template(node, bad_block, 'time-too-new') + bad_block.nTime = 0 + assert_template(node, bad_block, 'time-too-old') + + self.log.info("getblocktemplate: Test not best block") + bad_block = copy.deepcopy(block) + bad_block.hashPrevBlock = 123 + assert_template(node, bad_block, 'inconclusive-not-best-prevblk') + +if __name__ == '__main__': + MiningTest().main() diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py new file mode 100755 index 0000000000..fc6e8e325f --- /dev/null +++ b/test/functional/multiwallet.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test multiwallet. + +Verify that a bitcoind node can load multiple wallet files +""" +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_jsonrpc + +class MultiWalletTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3']] + + def run_test(self): + self.stop_node(0) + + # should not initialize if there are duplicate wallets + self.assert_start_raises_init_error(0, self.options.tmpdir, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') + + # should not initialize if wallet file is a directory + os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) + self.assert_start_raises_init_error(0, self.options.tmpdir, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') + + # should not initialize if wallet file is a symlink + os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) + self.assert_start_raises_init_error(0, self.options.tmpdir, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') + + self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0]) + + w1 = self.nodes[0].get_wallet_rpc("w1") + w2 = self.nodes[0].get_wallet_rpc("w2") + w3 = self.nodes[0].get_wallet_rpc("w3") + wallet_bad = self.nodes[0].get_wallet_rpc("bad") + + w1.generate(1) + + # accessing invalid wallet fails + assert_raises_jsonrpc(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) + + # accessing wallet RPC without using wallet endpoint fails + assert_raises_jsonrpc(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) + + # check w1 wallet balance + w1_info = w1.getwalletinfo() + assert_equal(w1_info['immature_balance'], 50) + w1_name = w1_info['walletname'] + assert_equal(w1_name, "w1") + + # check w2 wallet balance + w2_info = w2.getwalletinfo() + assert_equal(w2_info['immature_balance'], 0) + w2_name = w2_info['walletname'] + assert_equal(w2_name, "w2") + + w3_name = w3.getwalletinfo()['walletname'] + assert_equal(w3_name, "w3") + + assert_equal({"w1", "w2", "w3"}, {w1_name, w2_name, w3_name}) + + w1.generate(101) + assert_equal(w1.getbalance(), 100) + assert_equal(w2.getbalance(), 0) + assert_equal(w3.getbalance(), 0) + + w1.sendtoaddress(w2.getnewaddress(), 1) + w1.sendtoaddress(w3.getnewaddress(), 2) + w1.generate(1) + assert_equal(w2.getbalance(), 1) + assert_equal(w3.getbalance(), 2) + +if __name__ == '__main__': + MultiWalletTest().main() diff --git a/test/functional/net.py b/test/functional/net.py index 3ba3764cf9..1e63d38035 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -85,7 +85,7 @@ class NetTest(BitcoinTestFramework): added_nodes = self.nodes[0].getaddednodeinfo(ip_port) assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) - # check that a non-existant node returns an error + # check that a non-existent node returns an error assert_raises_jsonrpc(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py index 33b57ef33d..5611c876ae 100755 --- a/test/functional/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -9,7 +9,10 @@ received a VERACK. This test connects to a node and sends it a few messages, trying to intice it into sending us something it shouldn't. -""" + +Also test that nodes that send unsupported service bits to bitcoind are disconnected +and don't receive a VERACK. Unsupported service bits are currently 1 << 5 and +1 << 7 (until August 1st 2018).""" from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework @@ -98,20 +101,29 @@ class P2PLeakTest(BitcoinTestFramework): no_version_bannode = CNodeNoVersionBan() no_version_idlenode = CNodeNoVersionIdle() no_verack_idlenode = CNodeNoVerackIdle() + unsupported_service_bit5_node = CLazyNode() + unsupported_service_bit7_node = CLazyNode() + self.nodes[0].setmocktime(1501545600) # August 1st 2017 connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_bannode, send_version=False)) connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_idlenode, send_version=False)) connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_verack_idlenode)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) no_version_bannode.add_connection(connections[0]) no_version_idlenode.add_connection(connections[1]) no_verack_idlenode.add_connection(connections[2]) + unsupported_service_bit5_node.add_connection(connections[3]) + unsupported_service_bit7_node.add_connection(connections[4]) NetworkThread().start() # Start up network handling in another thread assert wait_until(lambda: no_version_bannode.ever_connected, timeout=10) assert wait_until(lambda: no_version_idlenode.ever_connected, timeout=10) assert wait_until(lambda: no_verack_idlenode.version_received, timeout=10) + assert wait_until(lambda: unsupported_service_bit5_node.ever_connected, timeout=10) + assert wait_until(lambda: unsupported_service_bit7_node.ever_connected, timeout=10) # Mine a block and make sure that it's not sent to the connected nodes self.nodes[0].generate(1) @@ -122,12 +134,32 @@ class P2PLeakTest(BitcoinTestFramework): #This node should have been banned assert not no_version_bannode.connected + # These nodes should have been disconnected + assert not unsupported_service_bit5_node.connected + assert not unsupported_service_bit7_node.connected + [conn.disconnect_node() for conn in connections] # Make sure no unexpected messages came in assert(no_version_bannode.unexpected_msg == False) assert(no_version_idlenode.unexpected_msg == False) assert(no_verack_idlenode.unexpected_msg == False) + assert not unsupported_service_bit5_node.unexpected_msg + assert not unsupported_service_bit7_node.unexpected_msg + + self.log.info("Service bits 5 and 7 are allowed after August 1st 2018") + self.nodes[0].setmocktime(1533168000) # August 2nd 2018 + + allowed_service_bit5_node = NodeConnCB() + allowed_service_bit7_node = NodeConnCB() + + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) + allowed_service_bit5_node.add_connection(connections[5]) + allowed_service_bit7_node.add_connection(connections[6]) + + assert wait_until(lambda: allowed_service_bit5_node.message_count["verack"], timeout=10) + assert wait_until(lambda: allowed_service_bit7_node.message_count["verack"], timeout=10) if __name__ == '__main__': P2PLeakTest().main() diff --git a/test/functional/pruning.py b/test/functional/pruning.py index 0af91e0658..3e00a34ac4 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -136,7 +136,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) self.nodes[1].invalidateblock(badhash) - # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want + # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) curhash = self.nodes[1].getblockhash(invalidheight - 1) @@ -199,7 +199,7 @@ class PruneTest(BitcoinTestFramework): goalbesthash = self.mainchainhash2 # As of 0.10 the current block download logic is not able to reorg to the original chain created in - # create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to + # create_chain_with_stale_blocks because it doesn't know of any peer that's on that chain from which to # redownload its missing blocks. # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain # because it has all the block data. diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index 35debf9cab..b6b90d6781 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -2,7 +2,7 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the rawtranscation RPCs. +"""Test the rawtransaction RPCs. Test the following RPCs: - createrawtransaction @@ -114,7 +114,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx = self.nodes[2].createrawtransaction(inputs, outputs) rawTxPartialSigned = self.nodes[1].signrawtransaction(rawTx, inputs) assert_equal(rawTxPartialSigned['complete'], False) #node1 only has one key, can't comp. sign the tx - + rawTxSigned = self.nodes[2].signrawtransaction(rawTx, inputs) assert_equal(rawTxSigned['complete'], True) #node2 can sign the tx compl., own two of three keys self.nodes[2].sendrawtransaction(rawTxSigned['hex']) @@ -124,6 +124,55 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + # 2of2 test for combining transactions + bal = self.nodes[2].getbalance() + addr1 = self.nodes[1].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[1].validateaddress(addr1) + addr2Obj = self.nodes[2].validateaddress(addr2) + + self.nodes[1].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) + mSigObjValid = self.nodes[2].validateaddress(mSigObj) + + txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) + decTx = self.nodes[0].gettransaction(txId) + rawTx2 = self.nodes[0].decoderawtransaction(decTx['hex']) + self.sync_all() + self.nodes[0].generate(1) + self.sync_all() + + assert_equal(self.nodes[2].getbalance(), bal) # the funds of a 2of2 multisig tx should not be marked as spendable + + txDetails = self.nodes[0].gettransaction(txId, True) + rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) + vout = False + for outpoint in rawTx2['vout']: + if outpoint['value'] == Decimal('2.20000000'): + vout = outpoint + break + + bal = self.nodes[0].getbalance() + inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex']}] + outputs = { self.nodes[0].getnewaddress() : 2.19 } + rawTx2 = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxPartialSigned1 = self.nodes[1].signrawtransaction(rawTx2, inputs) + self.log.info(rawTxPartialSigned1) + assert_equal(rawTxPartialSigned['complete'], False) #node1 only has one key, can't comp. sign the tx + + rawTxPartialSigned2 = self.nodes[2].signrawtransaction(rawTx2, inputs) + self.log.info(rawTxPartialSigned2) + assert_equal(rawTxPartialSigned2['complete'], False) #node2 only has one key, can't comp. sign the tx + rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) + self.log.info(rawTxComb) + self.nodes[2].sendrawtransaction(rawTxComb) + rawTx2 = self.nodes[0].decoderawtransaction(rawTxComb) + self.sync_all() + self.nodes[0].generate(1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + # getrawtransaction tests # 1. valid parameters - only supply txid txHash = rawTx["hash"] @@ -156,17 +205,17 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction(inputs, outputs) decrawtx= self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['vin'][0]['sequence'], 1000) - + # 9. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : -1}] outputs = { self.nodes[0].getnewaddress() : 1 } assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) - + # 10. invalid parameters - sequence number out of range inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967296}] outputs = { self.nodes[0].getnewaddress() : 1 } assert_raises_jsonrpc(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) - + inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967294}] outputs = { self.nodes[0].getnewaddress() : 1 } rawtx = self.nodes[0].createrawtransaction(inputs, outputs) diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index 2cad6269ac..19d99c9c9e 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -31,7 +31,7 @@ class ReceivedByTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime - enable_mocktime() + self.enable_mocktime() self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): diff --git a/test/functional/replace-by-fee.py b/test/functional/replace-by-fee.py index d6bf3ea59f..bc67654987 100755 --- a/test/functional/replace-by-fee.py +++ b/test/functional/replace-by-fee.py @@ -531,8 +531,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(json1["vin"][0]["sequence"], 4294967295) rawtx2 = self.nodes[0].createrawtransaction([], outs) - frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"optIntoRbf": True}) - frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"optIntoRbf": False}) + frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) + frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) json0 = self.nodes[0].decoderawtransaction(frawtx2a['hex']) json1 = self.nodes[0].decoderawtransaction(frawtx2b['hex']) diff --git a/test/functional/resendwallettransactions.py b/test/functional/resendwallettransactions.py new file mode 100755 index 0000000000..5059aa106e --- /dev/null +++ b/test/functional/resendwallettransactions.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test resendwallettransactions RPC.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_jsonrpc + +class ResendWalletTransactionsTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.extra_args = [['--walletbroadcast=false']] + self.num_nodes = 1 + + def run_test(self): + # Should raise RPC_WALLET_ERROR (-4) if walletbroadcast is disabled. + assert_raises_jsonrpc(-4, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast", self.nodes[0].resendwallettransactions) + + # Should return an empty array if there aren't unconfirmed wallet transactions. + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + assert_equal(self.nodes[0].resendwallettransactions(), []) + + # Should return an array with the unconfirmed wallet transaction. + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + assert_equal(self.nodes[0].resendwallettransactions(), [txid]) + +if __name__ == '__main__': + ResendWalletTransactionsTest().main() diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index 198599010e..20808207b2 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -37,7 +37,7 @@ class RPCBindTest(BitcoinTestFramework): base_args += ['-rpcallowip=' + x for x in allow_ips] binds = ['-rpcbind='+addr for addr in addresses] self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) - pid = bitcoind_processes[0].pid + pid = self.nodes[0].process.pid assert_equal(set(get_bind_addrs(pid)), set(expected)) self.stop_nodes() @@ -49,7 +49,7 @@ class RPCBindTest(BitcoinTestFramework): base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) # connect to node through non-loopback interface - node = get_rpc_proxy(rpc_url(get_datadir_path(self.options.tmpdir, 0), 0, "%s:%d" % (rpchost, rpcport)), 0) + node = get_rpc_proxy(rpc_url(get_datadir_path(self.options.tmpdir, 0), 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir) node.getnetworkinfo() self.stop_nodes() diff --git a/test/functional/segwit.py b/test/functional/segwit.py index ac95d66466..51eaa34a54 100755 --- a/test/functional/segwit.py +++ b/test/functional/segwit.py @@ -459,13 +459,14 @@ class SegWitTest(BitcoinTestFramework): self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) self.mine_and_test_listunspent(unseen_anytime, 0) - # addwitnessaddress should refuse to return a witness address if an uncompressed key is used or the address is - # not in the wallet + # addwitnessaddress should refuse to return a witness address if an uncompressed key is used # note that no witness address should be returned by unsolvable addresses - # the multisig_without_privkey_address will fail because its keys were not added with importpubkey - for i in uncompressed_spendable_address + uncompressed_solvable_address + unknown_address + unsolvable_address + [multisig_without_privkey_address]: + for i in uncompressed_spendable_address + uncompressed_solvable_address + unknown_address + unsolvable_address: assert_raises_jsonrpc(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) + # addwitnessaddress should return a witness addresses even if keys are not in the wallet + self.nodes[0].addwitnessaddress(multisig_without_privkey_address) + for i in compressed_spendable_address + compressed_solvable_address: witaddress = self.nodes[0].addwitnessaddress(i) # addwitnessaddress should return the same address if it is a known P2SH-witness address @@ -542,7 +543,7 @@ class SegWitTest(BitcoinTestFramework): # addwitnessaddress should refuse to return a witness address if an uncompressed key is used # note that a multisig address returned by addmultisigaddress is not solvable until it is added with importaddress # premature_witaddress are not accepted until the script is added with addwitnessaddress first - for i in uncompressed_spendable_address + uncompressed_solvable_address + premature_witaddress + [compressed_solvable_address[1]]: + for i in uncompressed_spendable_address + uncompressed_solvable_address + premature_witaddress: # This will raise an exception assert_raises_jsonrpc(-4, "Public key or redeemscript not known to wallet, or the key is uncompressed", self.nodes[0].addwitnessaddress, i) diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py index 44c357c6db..e47e07fb86 100755 --- a/test/functional/sendheaders.py +++ b/test/functional/sendheaders.py @@ -121,9 +121,6 @@ class TestNode(NodeConnCB): message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 - def on_block(self, conn, message): - self.last_message["block"].calc_sha256() - # Test whether the last announcement we received had the # right header or the right inv # inv and headers should be lists of block hashes diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index 437905e764..415727268a 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -43,22 +43,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): # 2) No script verification error occurred assert 'errors' not in rawTxSigned - # Check that signrawtransaction doesn't blow up on garbage merge attempts - dummyTxInconsistent = self.nodes[0].createrawtransaction([inputs[0]], outputs) - rawTxUnsigned = self.nodes[0].signrawtransaction(rawTx + dummyTxInconsistent, inputs) - - assert 'complete' in rawTxUnsigned - assert_equal(rawTxUnsigned['complete'], False) - - # Check that signrawtransaction properly merges unsigned and signed txn, even with garbage in the middle - rawTxSigned2 = self.nodes[0].signrawtransaction(rawTxUnsigned["hex"] + dummyTxInconsistent + rawTxSigned["hex"], inputs) - - assert 'complete' in rawTxSigned2 - assert_equal(rawTxSigned2['complete'], True) - - assert 'errors' not in rawTxSigned2 - - def script_verification_error_test(self): """Create and sign a raw transaction with valid (vin 0), invalid (vin 1) and one missing (vin 2) input script. diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index dfcc524313..b3671cbdc5 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -191,3 +191,6 @@ class AuthServiceProxy(object): else: log.debug("<-- [%.6f] %s"%(elapsed,responsedata)) return response + + def __truediv__(self, relative_uri): + return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 3f87ef91f6..227b1a17af 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -56,6 +56,8 @@ class AuthServiceProxyWrapper(object): def url(self): return self.auth_service_proxy_instance.url + def __truediv__(self, relative_uri): + return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri) def get_filename(dirname, n_node): """ diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 688347a68f..a4d85501e7 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -51,6 +51,8 @@ NODE_NETWORK = (1 << 0) NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) +NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) +NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) logger = logging.getLogger("TestFramework.mininode") diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index c7fd44b81c..7903bb0045 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -9,39 +9,30 @@ from enum import Enum import logging import optparse import os +import pdb import shutil -import subprocess import sys import tempfile import time +import traceback +from .authproxy import JSONRPCException +from . import coverage +from .test_node import TestNode from .util import ( - PortSeed, MAX_NODES, - bitcoind_processes, + PortSeed, + assert_equal, check_json_precision, connect_nodes_bi, - disable_mocktime, disconnect_nodes, - enable_coverage, - enable_mocktime, - get_mocktime, - get_rpc_proxy, initialize_datadir, - get_datadir_path, log_filename, p2p_port, - rpc_url, set_node_times, - _start_node, - _start_nodes, - _stop_node, - _stop_nodes, sync_blocks, sync_mempools, - wait_for_bitcoind_start, ) -from .authproxy import JSONRPCException class TestStatus(Enum): PASSED = 1 @@ -52,6 +43,8 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 +BITCOIND_PROC_WAIT_TIMEOUT = 60 + class BitcoinTestFramework(object): """Base class for a bitcoin test script. @@ -71,13 +64,14 @@ class BitcoinTestFramework(object): def __init__(self): self.num_nodes = 4 self.setup_clean_chain = False - self.nodes = None + self.nodes = [] + self.mocktime = 0 def add_options(self, parser): pass def setup_chain(self): - self.log.info("Initializing test directory "+self.options.tmpdir) + self.log.info("Initializing test directory " + self.options.tmpdir) if self.setup_clean_chain: self._initialize_chain_clean(self.options.tmpdir, self.num_nodes) else: @@ -97,7 +91,7 @@ class BitcoinTestFramework(object): extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args - self.nodes = _start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) def run_test(self): raise NotImplementedError @@ -111,9 +105,9 @@ class BitcoinTestFramework(object): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") - parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"), + parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../../src"), help="Source directory containing bitcoind/bitcoin-cli (default: %default)") - parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"), + parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs") parser.add_option("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO", @@ -126,15 +120,14 @@ class BitcoinTestFramework(object): help="Write tested RPC commands into this directory") parser.add_option("--configfile", dest="configfile", help="Location of the test framework config file") + parser.add_option("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", + help="Attach a python debugger if test fails") self.add_options(parser) (self.options, self.args) = parser.parse_args() - if self.options.coveragedir: - enable_coverage(self.options.coveragedir) - PortSeed.n = self.options.port_seed - os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] + os.environ['PATH'] = self.options.srcdir + ":" + self.options.srcdir + "/qt:" + os.environ['PATH'] check_json_precision() @@ -166,6 +159,10 @@ class BitcoinTestFramework(object): except KeyboardInterrupt as e: self.log.warning("Exiting after keyboard interrupt") + if success == TestStatus.FAILED and self.options.pdbonfailure: + print("Testcase failed. Attaching python debugger. Enter ? for help") + pdb.set_trace() + if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: @@ -188,7 +185,7 @@ class BitcoinTestFramework(object): for fn in filenames: try: with open(fn, 'r') as f: - print("From" , fn, ":") + print("From", fn, ":") print("".join(deque(f, MAX_LINES_TO_PRINT))) except OSError: print("Opening file %s failed." % fn) @@ -208,16 +205,88 @@ class BitcoinTestFramework(object): # Public helper methods. These can be accessed by the subclass test scripts. def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - return _start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr) + """Start a bitcoind and return RPC connection to it""" + + if extra_args is None: + extra_args = [] + if binary is None: + binary = os.getenv("BITCOIND", "bitcoind") + node = TestNode(i, dirname, extra_args, rpchost, timewait, binary, stderr, self.mocktime, coverage_dir=self.options.coveragedir) + node.start() + node.wait_for_rpc_connection() + + if self.options.coveragedir is not None: + coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) + + return node def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - return _start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary) + """Start multiple bitcoinds, return RPC connections to them""" + + if extra_args is None: + extra_args = [[]] * num_nodes + if binary is None: + binary = [None] * num_nodes + assert_equal(len(extra_args), num_nodes) + assert_equal(len(binary), num_nodes) + nodes = [] + try: + for i in range(num_nodes): + nodes.append(TestNode(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir)) + nodes[i].start() + for node in nodes: + node.wait_for_rpc_connection() + except: + # If one node failed to start, stop the others + self.stop_nodes() + raise + + if self.options.coveragedir is not None: + for node in nodes: + coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) - def stop_node(self, num_node): - _stop_node(self.nodes[num_node], num_node) + return nodes + + def stop_node(self, i): + """Stop a bitcoind test node""" + self.nodes[i].stop_node() + while not self.nodes[i].is_node_stopped(): + time.sleep(0.1) def stop_nodes(self): - _stop_nodes(self.nodes) + """Stop multiple bitcoind test nodes""" + for node in self.nodes: + # Issue RPC to stop nodes + node.stop_node() + + for node in self.nodes: + # Wait for nodes to stop + while not node.is_node_stopped(): + time.sleep(0.1) + + def assert_start_raises_init_error(self, i, dirname, extra_args=None, expected_msg=None): + with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: + try: + self.start_node(i, dirname, extra_args, stderr=log_stderr) + self.stop_node(i) + except Exception as e: + assert 'bitcoind exited' in str(e) # node must have shutdown + self.nodes[i].running = False + self.nodes[i].process = None + if expected_msg is not None: + log_stderr.seek(0) + stderr = log_stderr.read().decode('utf-8') + if expected_msg not in stderr: + raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr) + else: + if expected_msg is None: + assert_msg = "bitcoind should have exited with an error" + else: + assert_msg = "bitcoind should have exited with expected error " + expected_msg + raise AssertionError(assert_msg) + + def wait_for_node_exit(self, i, timeout): + self.nodes[i].process.wait(timeout) def split_network(self): """ @@ -242,6 +311,21 @@ class BitcoinTestFramework(object): sync_blocks(group) sync_mempools(group) + def enable_mocktime(self): + """Enable mocktime for the script. + + mocktime may be needed for scripts that use the cached version of the + blockchain. If the cached version of the blockchain is used without + mocktime then the mempools will not sync due to IBD. + + For backwared compatibility of the python scripts with previous + versions of the cache, this helper function sets mocktime to Jan 1, + 2014 + (201 * 10 * 60)""" + self.mocktime = 1388534400 + (201 * 10 * 60) + + def disable_mocktime(self): + self.mocktime = 0 + # Private helper methods. These should not be accessed by the subclass test scripts. def _start_logging(self): @@ -257,7 +341,7 @@ class BitcoinTestFramework(object): ll = int(self.options.loglevel) if self.options.loglevel.isdigit() else self.options.loglevel.upper() ch.setLevel(ll) # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted) - formatter = logging.Formatter(fmt = '%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + formatter = logging.Formatter(fmt='%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') formatter.converter = time.gmtime fh.setFormatter(formatter) ch.setFormatter(formatter) @@ -299,18 +383,13 @@ class BitcoinTestFramework(object): args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) - bitcoind_processes[i] = subprocess.Popen(args) - self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up") - wait_for_bitcoind_start(bitcoind_processes[i], datadir, i) - self.log.debug("initialize_chain: RPC successfully started") + self.nodes.append(TestNode(i, cachedir, extra_args=[], rpchost=None, timewait=None, binary=None, stderr=None, mocktime=self.mocktime, coverage_dir=None)) + self.nodes[i].args = args + self.nodes[i].start() - self.nodes = [] - for i in range(MAX_NODES): - try: - self.nodes.append(get_rpc_proxy(rpc_url(get_datadir_path(cachedir, i), i), i)) - except: - self.log.exception("Error connecting to node %d" % i) - sys.exit(1) + # Wait for RPC connections to be ready + for node in self.nodes: + node.wait_for_rpc_connection() # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. @@ -319,8 +398,8 @@ class BitcoinTestFramework(object): # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past - enable_mocktime() - block_time = get_mocktime() - (201 * 10 * 60) + self.enable_mocktime() + block_time = self.mocktime - (201 * 10 * 60) for i in range(2): for peer in range(4): for j in range(25): @@ -333,7 +412,7 @@ class BitcoinTestFramework(object): # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] - disable_mocktime() + self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(cachedir, i, "debug.log")) os.remove(log_filename(cachedir, i, "db.log")) @@ -354,18 +433,13 @@ class BitcoinTestFramework(object): for i in range(num_nodes): initialize_datadir(test_dir, i) -# Test framework for doing p2p comparison testing, which sets up some bitcoind -# binaries: -# 1 binary: test binary -# 2 binaries: 1 test binary, 1 ref binary -# n>2 binaries: 1 test binary, n-1 ref binaries - -class SkipTest(Exception): - """This exception is raised to skip a test""" - def __init__(self, message): - self.message = message - class ComparisonTestFramework(BitcoinTestFramework): + """Test framework for doing p2p comparison testing + + Sets up some bitcoind binaries: + - 1 binary: test binary + - 2 binaries: 1 test binary, 1 ref binary + - n>2 binaries: 1 test binary, n-1 ref binaries""" def __init__(self): super().__init__() @@ -381,8 +455,15 @@ class ComparisonTestFramework(BitcoinTestFramework): help="bitcoind binary to use for reference nodes (if any)") def setup_network(self): + extra_args = [['-whitelist=127.0.0.1']]*self.num_nodes + if hasattr(self, "extra_args"): + extra_args = self.extra_args self.nodes = self.start_nodes( - self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1']] * self.num_nodes, + self.num_nodes, self.options.tmpdir, extra_args, binary=[self.options.testbinary] + - [self.options.refbinary]*(self.num_nodes-1)) + [self.options.refbinary] * (self.num_nodes - 1)) + +class SkipTest(Exception): + """This exception is raised to skip a test""" + def __init__(self, message): + self.message = message diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py new file mode 100755 index 0000000000..66f89d43f4 --- /dev/null +++ b/test/functional/test_framework/test_node.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Class for bitcoind node under test""" + +import errno +import http.client +import logging +import os +import subprocess +import time + +from .util import ( + assert_equal, + get_rpc_proxy, + rpc_url, +) +from .authproxy import JSONRPCException + +class TestNode(): + """A class for representing a bitcoind node under test. + + This class contains: + + - state about the node (whether it's running, etc) + - a Python subprocess.Popen object representing the running process + - an RPC connection to the node + + To make things easier for the test writer, a bit of magic is happening under the covers. + Any unrecognised messages will be dispatched to the RPC connection.""" + + def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): + self.index = i + self.datadir = os.path.join(dirname, "node" + str(i)) + self.rpchost = rpchost + self.rpc_timeout = timewait + if binary is None: + self.binary = os.getenv("BITCOIND", "bitcoind") + else: + self.binary = binary + self.stderr = stderr + self.coverage_dir = coverage_dir + # Most callers will just need to add extra args to the standard list below. For those callers that need more flexibity, they can just set the args property directly. + self.extra_args = extra_args + self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i] + + self.running = False + self.process = None + self.rpc_connected = False + self.rpc = None + self.url = None + self.log = logging.getLogger('TestFramework.node%d' % i) + + def __getattr__(self, *args, **kwargs): + """Dispatches any unrecognised messages to the RPC connection.""" + assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" + return self.rpc.__getattr__(*args, **kwargs) + + def start(self): + """Start the node.""" + self.process = subprocess.Popen(self.args + self.extra_args, stderr=self.stderr) + self.running = True + self.log.debug("bitcoind started, waiting for RPC to come up") + + def wait_for_rpc_connection(self): + """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" + + # Wait for up to 10 seconds for the RPC server to respond + for _ in range(40): + assert not self.process.poll(), "bitcoind exited with status %i during initialization" % self.process.returncode + try: + self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, coveragedir=self.coverage_dir) + self.rpc.getblockcount() + # If the call to getblockcount() succeeds then the RPC connection is up + self.rpc_connected = True + self.url = self.rpc.url + self.log.debug("RPC successfully started") + return + except IOError as e: + if e.errno != errno.ECONNREFUSED: # Port not yet open? + raise # unknown IO error + except JSONRPCException as e: # Initialization phase + if e.error['code'] != -28: # RPC in warmup? + raise # unknown JSON RPC exception + except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting + if "No RPC credentials" not in str(e): + raise + time.sleep(0.25) + raise AssertionError("Unable to connect to bitcoind") + + def get_wallet_rpc(self, wallet_name): + assert self.rpc_connected + assert self.rpc + wallet_path = "wallet/%s" % wallet_name + return self.rpc / wallet_path + + def stop_node(self): + """Stop the node.""" + if not self.running: + return + self.log.debug("Stopping node") + try: + self.stop() + except http.client.CannotSendRequest: + self.log.exception("Unable to stop node.") + + def is_node_stopped(self): + """Checks whether the node has stopped. + + Returns True if the node has stopped. False otherwise. + This method is responsible for freeing resources (self.process).""" + if not self.running: + return True + return_code = self.process.poll() + if return_code is not None: + # process has stopped. Assert that it didn't return an error code. + assert_equal(return_code, 0) + self.running = False + self.process = None + self.log.debug("Node stopped") + return True + return False + + def node_encrypt_wallet(self, passphrase): + """"Encrypts the wallet. + + This causes bitcoind to shutdown, so this method takes + care of cleaning up resources.""" + self.encryptwallet(passphrase) + while not self.is_node_stopped(): + time.sleep(0.1) + self.rpc = None + self.rpc_connected = False diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index fa6388bf96..4098fd8615 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -4,30 +4,162 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" -import os -import sys - -from binascii import hexlify, unhexlify from base64 import b64encode +from binascii import hexlify, unhexlify from decimal import Decimal, ROUND_DOWN import json -import http.client +import logging +import os import random -import shutil -import subprocess -import tempfile -import time import re -import errno -import logging +import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException -COVERAGE_DIR = None - logger = logging.getLogger("TestFramework.utils") +# Assert functions +################## + +def assert_fee_amount(fee, tx_size, fee_per_kB): + """Assert the fee was in range""" + target_fee = tx_size * fee_per_kB / 1000 + if fee < target_fee: + raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) + # allow the wallet's estimation to be at most 2 bytes off + if fee > (tx_size + 2) * fee_per_kB / 1000: + raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) + +def assert_equal(thing1, thing2, *args): + if thing1 != thing2 or any(thing1 != arg for arg in args): + raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args)) + +def assert_greater_than(thing1, thing2): + if thing1 <= thing2: + raise AssertionError("%s <= %s" % (str(thing1), str(thing2))) + +def assert_greater_than_or_equal(thing1, thing2): + if thing1 < thing2: + raise AssertionError("%s < %s" % (str(thing1), str(thing2))) + +def assert_raises(exc, fun, *args, **kwds): + assert_raises_message(exc, None, fun, *args, **kwds) + +def assert_raises_message(exc, message, fun, *args, **kwds): + try: + fun(*args, **kwds) + except exc as e: + if message is not None and message not in e.error['message']: + raise AssertionError("Expected substring not found:" + e.error['message']) + except Exception as e: + raise AssertionError("Unexpected exception raised: " + type(e).__name__) + else: + raise AssertionError("No exception raised") + +def assert_raises_jsonrpc(code, message, fun, *args, **kwds): + """Run an RPC and verify that a specific JSONRPC exception code and message is raised. + + Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException + and verifies that the error code and message are as expected. Throws AssertionError if + no JSONRPCException was returned or if the error code/message are not as expected. + + Args: + code (int), optional: the error code returned by the RPC call (defined + in src/rpc/protocol.h). Set to None if checking the error code is not required. + message (string), optional: [a substring of] the error string returned by the + RPC call. Set to None if checking the error string is not required + fun (function): the function to call. This should be the name of an RPC. + args*: positional arguments for the function. + kwds**: named arguments for the function. + """ + try: + fun(*args, **kwds) + except JSONRPCException as e: + # JSONRPCException was thrown as expected. Check the code and message values are correct. + if (code is not None) and (code != e.error["code"]): + raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) + if (message is not None) and (message not in e.error['message']): + raise AssertionError("Expected substring not found:" + e.error['message']) + except Exception as e: + raise AssertionError("Unexpected exception raised: " + type(e).__name__) + else: + raise AssertionError("No exception raised") + +def assert_is_hex_string(string): + try: + int(string, 16) + except Exception as e: + raise AssertionError( + "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) + +def assert_is_hash_string(string, length=64): + if not isinstance(string, str): + raise AssertionError("Expected a string, got type %r" % type(string)) + elif length and len(string) != length: + raise AssertionError( + "String of length %d expected; got %d" % (length, len(string))) + elif not re.match('[abcdef0-9]+$', string): + raise AssertionError( + "String %r contains invalid characters for a hash." % string) + +def assert_array_result(object_array, to_match, expected, should_not_find=False): + """ + Pass in array of JSON objects, a dictionary with key/value pairs + to match against, and another dictionary with expected key/value + pairs. + If the should_not_find flag is true, to_match should not be found + in object_array + """ + if should_not_find: + assert_equal(expected, {}) + num_matched = 0 + for item in object_array: + all_match = True + for key, value in to_match.items(): + if item[key] != value: + all_match = False + if not all_match: + continue + elif should_not_find: + num_matched = num_matched + 1 + for key, value in expected.items(): + if item[key] != value: + raise AssertionError("%s : expected %s=%s" % (str(item), str(key), str(value))) + num_matched = num_matched + 1 + if num_matched == 0 and not should_not_find: + raise AssertionError("No objects matched %s" % (str(to_match))) + if num_matched > 0 and should_not_find: + raise AssertionError("Objects were found %s" % (str(to_match))) + +# Utility functions +################### + +def check_json_precision(): + """Make sure json library being used does not lose precision converting BTC values""" + n = Decimal("20000000.00000003") + satoshis = int(json.loads(json.dumps(float(n))) * 1.0e8) + if satoshis != 2000000000000003: + raise RuntimeError("JSON encode/decode loses precision") + +def count_bytes(hex_string): + return len(bytearray.fromhex(hex_string)) + +def bytes_to_hex_str(byte_str): + return hexlify(byte_str).decode('ascii') + +def hex_str_to_bytes(hex_str): + return unhexlify(hex_str.encode('ascii')) + +def str_to_b64str(string): + return b64encode(string.encode('utf-8')).decode('ascii') + +def satoshi_round(amount): + return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + +# RPC/P2P connection constants and functions +############################################ + # The maximum number of nodes a single test can spawn MAX_NODES = 8 # Don't assign rpc or p2p ports lower than this @@ -35,41 +167,11 @@ PORT_MIN = 11000 # The number of ports to "reserve" for p2p and rpc, each PORT_RANGE = 5000 -BITCOIND_PROC_WAIT_TIMEOUT = 60 - - class PortSeed: # Must be initialized with a unique integer for each process n = None -#Set Mocktime default to OFF. -#MOCKTIME is only needed for scripts that use the -#cached version of the blockchain. If the cached -#version of the blockchain is used without MOCKTIME -#then the mempools will not sync due to IBD. -MOCKTIME = 0 - -def enable_mocktime(): - #For backwared compatibility of the python scripts - #with previous versions of the cache, set MOCKTIME - #to Jan 1, 2014 + (201 * 10 * 60) - global MOCKTIME - MOCKTIME = 1388534400 + (201 * 10 * 60) - -def disable_mocktime(): - global MOCKTIME - MOCKTIME = 0 - -def get_mocktime(): - return MOCKTIME - -def enable_coverage(dirname): - """Maintain a log of which RPC calls are made during testing.""" - global COVERAGE_DIR - COVERAGE_DIR = dirname - - -def get_rpc_proxy(url, node_number, timeout=None): +def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): """ Args: url (str): URL of the RPC server to call @@ -90,11 +192,10 @@ def get_rpc_proxy(url, node_number, timeout=None): proxy.url = url # store URL on proxy for info coverage_logfile = coverage.get_filename( - COVERAGE_DIR, node_number) if COVERAGE_DIR else None + coveragedir, node_number) if coveragedir else None return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) - def p2p_port(n): assert(n <= MAX_NODES) return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) @@ -102,106 +203,46 @@ def p2p_port(n): def rpc_port(n): return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) -def check_json_precision(): - """Make sure json library being used does not lose precision converting BTC values""" - n = Decimal("20000000.00000003") - satoshis = int(json.loads(json.dumps(float(n)))*1.0e8) - if satoshis != 2000000000000003: - raise RuntimeError("JSON encode/decode loses precision") - -def count_bytes(hex_string): - return len(bytearray.fromhex(hex_string)) - -def bytes_to_hex_str(byte_str): - return hexlify(byte_str).decode('ascii') - -def hex_str_to_bytes(hex_str): - return unhexlify(hex_str.encode('ascii')) - -def str_to_b64str(string): - return b64encode(string.encode('utf-8')).decode('ascii') - -def sync_blocks(rpc_connections, *, wait=1, timeout=60): - """ - Wait until everybody has the same tip. - - sync_blocks needs to be called with an rpc_connections set that has least - one node already synced to the latest, stable tip, otherwise there's a - chance it might return before all nodes are stably synced. - """ - # Use getblockcount() instead of waitforblockheight() to determine the - # initial max height because the two RPCs look at different internal global - # variables (chainActive vs latestBlock) and the former gets updated - # earlier. - maxheight = max(x.getblockcount() for x in rpc_connections) - start_time = cur_time = time.time() - while cur_time <= start_time + timeout: - tips = [r.waitforblockheight(maxheight, int(wait * 1000)) for r in rpc_connections] - if all(t["height"] == maxheight for t in tips): - if all(t["hash"] == tips[0]["hash"] for t in tips): - return - raise AssertionError("Block sync failed, mismatched block hashes:{}".format( - "".join("\n {!r}".format(tip) for tip in tips))) - cur_time = time.time() - raise AssertionError("Block sync to height {} timed out:{}".format( - maxheight, "".join("\n {!r}".format(tip) for tip in tips))) - -def sync_chain(rpc_connections, *, wait=1, timeout=60): - """ - Wait until everybody has the same best block - """ - while timeout > 0: - best_hash = [x.getbestblockhash() for x in rpc_connections] - if best_hash == [best_hash[0]]*len(best_hash): - return - time.sleep(wait) - timeout -= wait - raise AssertionError("Chain sync failed: Best block hashes don't match") - -def sync_mempools(rpc_connections, *, wait=1, timeout=60): - """ - Wait until everybody has the same transactions in their memory - pools - """ - while timeout > 0: - pool = set(rpc_connections[0].getrawmempool()) - num_match = 1 - for i in range(1, len(rpc_connections)): - if set(rpc_connections[i].getrawmempool()) == pool: - num_match = num_match+1 - if num_match == len(rpc_connections): - return - time.sleep(wait) - timeout -= wait - raise AssertionError("Mempool sync failed") +def rpc_url(datadir, i, rpchost=None): + rpc_u, rpc_p = get_auth_cookie(datadir) + host = '127.0.0.1' + port = rpc_port(i) + if rpchost: + parts = rpchost.split(':') + if len(parts) == 2: + host, port = parts + else: + host = rpchost + return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) -bitcoind_processes = {} +# Node functions +################ def initialize_datadir(dirname, n): - datadir = os.path.join(dirname, "node"+str(n)) + datadir = os.path.join(dirname, "node" + str(n)) if not os.path.isdir(datadir): os.makedirs(datadir) with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: f.write("regtest=1\n") - f.write("port="+str(p2p_port(n))+"\n") - f.write("rpcport="+str(rpc_port(n))+"\n") + f.write("port=" + str(p2p_port(n)) + "\n") + f.write("rpcport=" + str(rpc_port(n)) + "\n") f.write("listenonion=0\n") return datadir def get_datadir_path(dirname, n): - return os.path.join(dirname, "node"+str(n)) + return os.path.join(dirname, "node" + str(n)) -def get_auth_cookie(datadir, n): +def get_auth_cookie(datadir): user = None password = None if os.path.isfile(os.path.join(datadir, "bitcoin.conf")): - with open(os.path.join(datadir, "bitcoin.conf"), 'r') as f: + with open(os.path.join(datadir, "bitcoin.conf"), 'r', encoding='utf8') as f: for line in f: if line.startswith("rpcuser="): - assert user is None # Ensure that there is only one rpcuser line + assert user is None # Ensure that there is only one rpcuser line user = line.split("=")[1].strip("\n") if line.startswith("rpcpassword="): - assert password is None # Ensure that there is only one rpcpassword line + assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: @@ -213,126 +254,12 @@ def get_auth_cookie(datadir, n): raise ValueError("No RPC credentials") return user, password -def rpc_url(datadir, i, rpchost=None): - rpc_u, rpc_p = get_auth_cookie(datadir, i) - host = '127.0.0.1' - port = rpc_port(i) - if rpchost: - parts = rpchost.split(':') - if len(parts) == 2: - host, port = parts - else: - host = rpchost - return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) - -def wait_for_bitcoind_start(process, datadir, i, rpchost=None): - ''' - Wait for bitcoind to start. This means that RPC is accessible and fully initialized. - Raise an exception if bitcoind exits during initialization. - ''' - while True: - if process.poll() is not None: - raise Exception('bitcoind exited with status %i during initialization' % process.returncode) - try: - # Check if .cookie file to be created - rpc = get_rpc_proxy(rpc_url(datadir, i, rpchost), i) - blocks = rpc.getblockcount() - break # break out of loop on success - except IOError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? - raise # unknown IO error - except JSONRPCException as e: # Initialization phase - if e.error['code'] != -28: # RPC in warmup? - raise # unknown JSON RPC exception - except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting - if "No RPC credentials" not in str(e): - raise - time.sleep(0.25) - - -def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - """Start a bitcoind and return RPC connection to it - - This function should only be called from within test_framework, not by individual test scripts.""" - - datadir = os.path.join(dirname, "node"+str(i)) - if binary is None: - binary = os.getenv("BITCOIND", "bitcoind") - args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i] - if extra_args is not None: args.extend(extra_args) - bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) - logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") - wait_for_bitcoind_start(bitcoind_processes[i], datadir, i, rpchost) - logger.debug("initialize_chain: RPC successfully started") - proxy = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, timeout=timewait) - - if COVERAGE_DIR: - coverage.write_all_rpc_commands(COVERAGE_DIR, proxy) - - return proxy - -def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): - with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: - try: - node = _start_node(i, dirname, extra_args, stderr=log_stderr) - _stop_node(node, i) - except Exception as e: - assert 'bitcoind exited' in str(e) #node must have shutdown - if expected_msg is not None: - log_stderr.seek(0) - stderr = log_stderr.read().decode('utf-8') - if expected_msg not in stderr: - raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr) - else: - if expected_msg is None: - assert_msg = "bitcoind should have exited with an error" - else: - assert_msg = "bitcoind should have exited with expected error " + expected_msg - raise AssertionError(assert_msg) - -def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """Start multiple bitcoinds, return RPC connections to them - - This function should only be called from within test_framework, not by individual test scripts.""" - - if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] - if binary is None: binary = [ None for _ in range(num_nodes) ] - assert_equal(len(extra_args), num_nodes) - assert_equal(len(binary), num_nodes) - rpcs = [] - try: - for i in range(num_nodes): - rpcs.append(_start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) - except: # If one node failed to start, stop the others - _stop_nodes(rpcs) - raise - return rpcs - def log_filename(dirname, n_node, logname): - return os.path.join(dirname, "node"+str(n_node), "regtest", logname) - -def _stop_node(node, i): - """Stop a bitcoind test node - - This function should only be called from within test_framework, not by individual test scripts.""" - - logger.debug("Stopping node %d" % i) - try: - node.stop() - except http.client.CannotSendRequest as e: - logger.exception("Unable to stop node") - return_code = bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - del bitcoind_processes[i] - assert_equal(return_code, 0) - -def _stop_nodes(nodes): - """Stop multiple bitcoind test nodes - - This function should only be called from within test_framework, not by individual test scripts.""" + return os.path.join(dirname, "node" + str(n_node), "regtest", logname) - for i, node in enumerate(nodes): - _stop_node(node, i) - assert not bitcoind_processes.values() # All connections must be gone now +def get_bip9_status(node, key): + info = node.getblockchaininfo() + return info['bip9_softforks'][key] def set_node_times(nodes, t): for node in nodes: @@ -350,7 +277,7 @@ def disconnect_nodes(from_connection, node_num): raise AssertionError("timed out waiting for disconnect") def connect_nodes(from_connection, node_num): - ip_port = "127.0.0.1:"+str(p2p_port(node_num)) + ip_port = "127.0.0.1:" + str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") # poll until version handshake complete to avoid race conditions # with transaction relaying @@ -361,6 +288,63 @@ def connect_nodes_bi(nodes, a, b): connect_nodes(nodes[a], b) connect_nodes(nodes[b], a) +def sync_blocks(rpc_connections, *, wait=1, timeout=60): + """ + Wait until everybody has the same tip. + + sync_blocks needs to be called with an rpc_connections set that has least + one node already synced to the latest, stable tip, otherwise there's a + chance it might return before all nodes are stably synced. + """ + # Use getblockcount() instead of waitforblockheight() to determine the + # initial max height because the two RPCs look at different internal global + # variables (chainActive vs latestBlock) and the former gets updated + # earlier. + maxheight = max(x.getblockcount() for x in rpc_connections) + start_time = cur_time = time.time() + while cur_time <= start_time + timeout: + tips = [r.waitforblockheight(maxheight, int(wait * 1000)) for r in rpc_connections] + if all(t["height"] == maxheight for t in tips): + if all(t["hash"] == tips[0]["hash"] for t in tips): + return + raise AssertionError("Block sync failed, mismatched block hashes:{}".format( + "".join("\n {!r}".format(tip) for tip in tips))) + cur_time = time.time() + raise AssertionError("Block sync to height {} timed out:{}".format( + maxheight, "".join("\n {!r}".format(tip) for tip in tips))) + +def sync_chain(rpc_connections, *, wait=1, timeout=60): + """ + Wait until everybody has the same best block + """ + while timeout > 0: + best_hash = [x.getbestblockhash() for x in rpc_connections] + if best_hash == [best_hash[0]] * len(best_hash): + return + time.sleep(wait) + timeout -= wait + raise AssertionError("Chain sync failed: Best block hashes don't match") + +def sync_mempools(rpc_connections, *, wait=1, timeout=60): + """ + Wait until everybody has the same transactions in their memory + pools + """ + while timeout > 0: + pool = set(rpc_connections[0].getrawmempool()) + num_match = 1 + for i in range(1, len(rpc_connections)): + if set(rpc_connections[i].getrawmempool()) == pool: + num_match = num_match + 1 + if num_match == len(rpc_connections): + return + time.sleep(wait) + timeout -= wait + raise AssertionError("Mempool sync failed") + +# Transaction/Block functions +############################# + def find_output(node, txid, amount): """ Return index to output of txid with value amount @@ -370,14 +354,13 @@ def find_output(node, txid, amount): for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i - raise RuntimeError("find_output txid %s : %s not found"%(txid,str(amount))) - + raise RuntimeError("find_output txid %s : %s not found" % (txid, str(amount))) def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ - assert(confirmations_required >=0) + assert(confirmations_required >= 0) utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] @@ -385,9 +368,9 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): while total_in < amount_needed and len(utxo) > 0: t = utxo.pop() total_in += t["amount"] - inputs.append({ "txid" : t["txid"], "vout" : t["vout"], "address" : t["address"] } ) + inputs.append({"txid": t["txid"], "vout": t["vout"], "address": t["address"]}) if total_in < amount_needed: - raise RuntimeError("Insufficient funds: need %d, have %d"%(amount_needed, total_in)) + raise RuntimeError("Insufficient funds: need %d, have %d" % (amount_needed, total_in)) return (total_in, inputs) def make_change(from_node, amount_in, amount_out, fee): @@ -395,13 +378,13 @@ def make_change(from_node, amount_in, amount_out, fee): Create change output(s), return them """ outputs = {} - amount = amount_out+fee + amount = amount_out + fee change = amount_in - amount - if change > amount*2: + if change > amount * 2: # Create an extra change output to break up big inputs change_address = from_node.getnewaddress() # Split change in two, being careful of rounding: - outputs[change_address] = Decimal(change/2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + outputs[change_address] = Decimal(change / 2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) change = amount_in - amount - outputs[change_address] if change > 0: outputs[from_node.getnewaddress()] = change @@ -414,9 +397,9 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ from_node = random.choice(nodes) to_node = random.choice(nodes) - fee = min_fee + fee_increment*random.randint(0,fee_variants) + fee = min_fee + fee_increment * random.randint(0, fee_variants) - (total_in, inputs) = gather_inputs(from_node, amount+fee) + (total_in, inputs) = gather_inputs(from_node, amount + fee) outputs = make_change(from_node, total_in, amount, fee) outputs[to_node.getnewaddress()] = float(amount) @@ -426,123 +409,13 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): return (txid, signresult["hex"], fee) -def assert_fee_amount(fee, tx_size, fee_per_kB): - """Assert the fee was in range""" - target_fee = tx_size * fee_per_kB / 1000 - if fee < target_fee: - raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)"%(str(fee), str(target_fee))) - # allow the wallet's estimation to be at most 2 bytes off - if fee > (tx_size + 2) * fee_per_kB / 1000: - raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)"%(str(fee), str(target_fee))) - -def assert_equal(thing1, thing2, *args): - if thing1 != thing2 or any(thing1 != arg for arg in args): - raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args)) - -def assert_greater_than(thing1, thing2): - if thing1 <= thing2: - raise AssertionError("%s <= %s"%(str(thing1),str(thing2))) - -def assert_greater_than_or_equal(thing1, thing2): - if thing1 < thing2: - raise AssertionError("%s < %s"%(str(thing1),str(thing2))) - -def assert_raises(exc, fun, *args, **kwds): - assert_raises_message(exc, None, fun, *args, **kwds) - -def assert_raises_message(exc, message, fun, *args, **kwds): - try: - fun(*args, **kwds) - except exc as e: - if message is not None and message not in e.error['message']: - raise AssertionError("Expected substring not found:"+e.error['message']) - except Exception as e: - raise AssertionError("Unexpected exception raised: "+type(e).__name__) - else: - raise AssertionError("No exception raised") - -def assert_raises_jsonrpc(code, message, fun, *args, **kwds): - """Run an RPC and verify that a specific JSONRPC exception code and message is raised. - - Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException - and verifies that the error code and message are as expected. Throws AssertionError if - no JSONRPCException was returned or if the error code/message are not as expected. - - Args: - code (int), optional: the error code returned by the RPC call (defined - in src/rpc/protocol.h). Set to None if checking the error code is not required. - message (string), optional: [a substring of] the error string returned by the - RPC call. Set to None if checking the error string is not required - fun (function): the function to call. This should be the name of an RPC. - args*: positional arguments for the function. - kwds**: named arguments for the function. - """ - try: - fun(*args, **kwds) - except JSONRPCException as e: - # JSONRPCException was thrown as expected. Check the code and message values are correct. - if (code is not None) and (code != e.error["code"]): - raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) - if (message is not None) and (message not in e.error['message']): - raise AssertionError("Expected substring not found:"+e.error['message']) - except Exception as e: - raise AssertionError("Unexpected exception raised: "+type(e).__name__) - else: - raise AssertionError("No exception raised") - -def assert_is_hex_string(string): - try: - int(string, 16) - except Exception as e: - raise AssertionError( - "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) - -def assert_is_hash_string(string, length=64): - if not isinstance(string, str): - raise AssertionError("Expected a string, got type %r" % type(string)) - elif length and len(string) != length: - raise AssertionError( - "String of length %d expected; got %d" % (length, len(string))) - elif not re.match('[abcdef0-9]+$', string): - raise AssertionError( - "String %r contains invalid characters for a hash." % string) - -def assert_array_result(object_array, to_match, expected, should_not_find = False): - """ - Pass in array of JSON objects, a dictionary with key/value pairs - to match against, and another dictionary with expected key/value - pairs. - If the should_not_find flag is true, to_match should not be found - in object_array - """ - if should_not_find == True: - assert_equal(expected, { }) - num_matched = 0 - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - elif should_not_find == True: - num_matched = num_matched+1 - for key,value in expected.items(): - if item[key] != value: - raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) - num_matched = num_matched+1 - if num_matched == 0 and should_not_find != True: - raise AssertionError("No objects matched %s"%(str(to_match))) - if num_matched > 0 and should_not_find == True: - raise AssertionError("Objects were found %s"%(str(to_match))) - -def satoshi_round(amount): - return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) - # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. def create_confirmed_utxos(fee, node, count): - node.generate(int(0.5*count)+101) + to_generate = int(0.5 * count) + 101 + while to_generate > 0: + node.generate(min(25, to_generate)) + to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) addr1 = node.getnewaddress() @@ -552,14 +425,14 @@ def create_confirmed_utxos(fee, node, count): for i in range(iterations): t = utxos.pop() inputs = [] - inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) + inputs.append({"txid": t["txid"], "vout": t["vout"]}) outputs = {} send_value = t['amount'] - fee - outputs[addr1] = satoshi_round(send_value/2) - outputs[addr2] = satoshi_round(send_value/2) + outputs[addr1] = satoshi_round(send_value / 2) + outputs[addr2] = satoshi_round(send_value / 2) raw_tx = node.createrawtransaction(inputs, outputs) signed_tx = node.signrawtransaction(raw_tx)["hex"] - txid = node.sendrawtransaction(signed_tx) + node.sendrawtransaction(signed_tx) while (node.getmempoolinfo()['size'] > 0): node.generate(1) @@ -574,8 +447,8 @@ def gen_return_txouts(): # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create # So we have big transactions (and therefore can't fit very many into each block) # create one script_pubkey - script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes - for i in range (512): + script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes + for i in range(512): script_pubkey = script_pubkey + "01" # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change txouts = "81" @@ -589,8 +462,8 @@ def gen_return_txouts(): return txouts def create_tx(node, coinbase, to_address, amount): - inputs = [{ "txid" : coinbase, "vout" : 0}] - outputs = { to_address : amount } + inputs = [{"txid": coinbase, "vout": 0}] + outputs = {to_address: amount} rawtx = node.createrawtransaction(inputs, outputs) signresult = node.signrawtransaction(rawtx) assert_equal(signresult["complete"], True) @@ -603,7 +476,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): txids = [] for _ in range(num): t = utxos.pop() - inputs=[{ "txid" : t["txid"], "vout" : t["vout"]}] + inputs = [{"txid": t["txid"], "vout": t["vout"]}] outputs = {} change = t['amount'] - fee outputs[addr] = satoshi_round(change) @@ -628,7 +501,3 @@ def mine_large_block(node, utxos=None): fee = 100 * node.getnetworkinfo()["relayfee"] create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) node.generate(1) - -def get_bip9_status(node, key): - info = node.getblockchaininfo() - return info['bip9_softforks'][key] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index ffa5b0ae20..d248a6c005 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -79,6 +79,7 @@ BASE_SCRIPTS= [ 'rawtransactions.py', 'reindex.py', # vv Tests less than 30s vv + 'keypool-topup.py', 'zmq_test.py', 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', @@ -88,6 +89,7 @@ BASE_SCRIPTS= [ 'mempool_spendcoinbase.py', 'mempool_reorg.py', 'mempool_persist.py', + 'multiwallet.py', 'httpbasics.py', 'multi_rpc.py', 'proxy_test.py', @@ -108,11 +110,16 @@ BASE_SCRIPTS= [ 'signmessages.py', 'nulldummy.py', 'import-rescan.py', + 'mining.py', 'bumpfee.py', 'rpcnamedargs.py', 'listsinceblock.py', 'p2p-leaktests.py', 'wallet-encryption.py', + 'bipdersig-p2p.py', + 'bip65-cltv-p2p.py', + 'uptime.py', + 'resendwallettransactions.py', ] EXTENDED_SCRIPTS = [ @@ -124,6 +131,7 @@ EXTENDED_SCRIPTS = [ # vv Tests less than 5m vv 'maxuploadtarget.py', 'mempool_packages.py', + 'dbcrash.py', # vv Tests less than 2m vv 'bip68-sequence.py', 'getblocktemplate_longpoll.py', @@ -134,11 +142,7 @@ EXTENDED_SCRIPTS = [ 'rpcbind_test.py', # vv Tests less than 30s vv 'assumevalid.py', - 'bip65-cltv.py', - 'bip65-cltv-p2p.py', - 'bipdersig-p2p.py', - 'bipdersig.py', - 'getblocktemplate_proposals.py', + 'example_test.py', 'txn_doublespend.py', 'txn_clone.py --mineblock', 'forknotify.py', @@ -166,7 +170,7 @@ def main(): Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') - parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.') + parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') diff --git a/test/functional/uptime.py b/test/functional/uptime.py new file mode 100755 index 0000000000..b20d6f5cb6 --- /dev/null +++ b/test/functional/uptime.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the RPC call related to the uptime command. + +Test corresponds to code in rpc/server.cpp. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework + + +class UptimeTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self._test_uptime() + + def _test_uptime(self): + wait_time = 10 + self.nodes[0].setmocktime(int(time.time() + wait_time)) + assert(self.nodes[0].uptime() >= wait_time) + + +if __name__ == '__main__': + UptimeTest().main() diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 9cb32d4650..61ad00330b 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -7,7 +7,7 @@ import os from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_equal, bitcoind_processes) +from test_framework.util import assert_equal def read_dump(file_name, addrs, hd_master_addr_old): @@ -94,8 +94,7 @@ class WalletDumpTest(BitcoinTestFramework): assert_equal(found_addr_rsv, 90*2) # 90 keys plus 100% internal keys #encrypt wallet, restart, unlock and dump - self.nodes[0].encryptwallet('test') - bitcoind_processes[0].wait() + self.nodes[0].node_encrypt_wallet('test') self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0]) self.nodes[0].walletpassphrase('test', 10) # Should be a no-op: diff --git a/test/functional/wallet-encryption.py b/test/functional/wallet-encryption.py index 33872e3c94..8fea4140db 100755 --- a/test/functional/wallet-encryption.py +++ b/test/functional/wallet-encryption.py @@ -6,12 +6,10 @@ import time -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT from test_framework.util import ( assert_equal, assert_raises_jsonrpc, - bitcoind_processes, - BITCOIND_PROC_WAIT_TIMEOUT, ) class WalletEncryptionTest(BitcoinTestFramework): @@ -32,8 +30,7 @@ class WalletEncryptionTest(BitcoinTestFramework): assert_equal(len(privkey), 52) # Encrypt the wallet - self.nodes[0].encryptwallet(passphrase) - bitcoind_processes[0].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) + self.nodes[0].node_encrypt_wallet(passphrase) self.nodes[0] = self.start_node(0, self.options.tmpdir) # Test that the wallet is encrypted diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index e7ec72a248..821575ed19 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -8,9 +8,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, - assert_start_raises_init_error ) -import os import shutil @@ -27,7 +25,7 @@ class WalletHDTest(BitcoinTestFramework): # Make sure can't switch off usehd after wallet creation self.stop_node(1) - assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') + self.assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) @@ -73,10 +71,12 @@ class WalletHDTest(BitcoinTestFramework): self.log.info("Restore backup ...") self.stop_node(1) - os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") + # we need to delete the complete regtest directory + # otherwise node1 would auto-recover all funds in flag the keypool keys as used + shutil.rmtree(tmpdir + "/node1/regtest/blocks") + shutil.rmtree(tmpdir + "/node1/regtest/chainstate") shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) - #connect_nodes_bi(self.nodes, 0, 1) # Assert that derivation is deterministic hd_add_2 = None @@ -86,11 +86,12 @@ class WalletHDTest(BitcoinTestFramework): assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(_+1)+"'") assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid) assert_equal(hd_add, hd_add_2) + connect_nodes_bi(self.nodes, 0, 1) + self.sync_all() # Needs rescan self.stop_node(1) self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan']) - #connect_nodes_bi(self.nodes, 0, 1) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) # send a tx and make sure its using the internal chain for the changeoutput diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index a4507182a2..ff51cba4b3 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -30,10 +30,11 @@ confirm 1/2/3/4 balances are same as before. Shutdown again, restore using importwallet, and confirm again balances are correct. """ +from random import randint +import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from random import randint class WalletBackupTest(BitcoinTestFramework): diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index e4d40520ef..af867d7a52 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -4,77 +4,73 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the zapwallettxes functionality. -- start three bitcoind nodes -- create four transactions on node 0 - two are confirmed and two are - unconfirmed. -- restart node 1 and verify that both the confirmed and the unconfirmed +- start two bitcoind nodes +- create two transactions on node 0 - one is confirmed and one is unconfirmed. +- restart node 0 and verify that both the confirmed and the unconfirmed transactions are still available. -- restart node 0 and verify that the confirmed transactions are still - available, but that the unconfirmed transaction has been zapped. +- restart node 0 with zapwallettxes and persistmempool, and verify that both + the confirmed and the unconfirmed transactions are still available. +- restart node 0 with just zapwallettxed and verify that the confirmed + transactions are still available, but that the unconfirmed transaction has + been zapped. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + ) class ZapWalletTXesTest (BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 3 - - def setup_network(self): - super().setup_network() - connect_nodes_bi(self.nodes,0,2) + self.num_nodes = 2 - def run_test (self): + def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(1) self.sync_all() - self.nodes[1].generate(101) - self.sync_all() - - assert_equal(self.nodes[0].getbalance(), 50) - - txid0 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid1 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + self.nodes[1].generate(100) self.sync_all() + + # This transaction will be confirmed + txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 10) + self.nodes[0].generate(1) self.sync_all() - - txid2 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid3 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 must be available (confirmed) - - tx1 = self.nodes[0].gettransaction(txid1) - assert_equal(tx1['txid'], txid1) #tx1 must be available (confirmed) - - tx2 = self.nodes[0].gettransaction(txid2) - assert_equal(tx2['txid'], txid2) #tx2 must be available (unconfirmed) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) - - #restart bitcoind + + # This transaction will not be confirmed + txid2 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 20) + + # Confirmed and unconfirmed transactions are now in the wallet. + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop-start node0. Both confirmed and unconfirmed transactions remain in the wallet. + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes and persistmempool. The unconfirmed + # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. self.stop_node(0) - self.nodes[0] = self.start_node(0,self.options.tmpdir) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-persistmempool=1", "-zapwallettxes=2"]) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes, but not persistmempool. + # The unconfirmed transaction is zapped and is no longer in the wallet. self.stop_node(0) - - #restart bitcoind with zapwallettxes - self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) - - assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) - #there must be an exception because the unconfirmed wallettx0 must be gone by now + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-zapwallettxes=2"]) - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 (confirmed) must still be available because it was confirmed + # tx1 is still be available because it was confirmed + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + # This will raise an exception because the unconfirmed transaction has been zapped + assert_raises_jsonrpc(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) if __name__ == '__main__': - ZapWalletTXesTest ().main () + ZapWalletTXesTest().main() diff --git a/test/util/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index f6dfbb51cc..0b3235dd4a 100644 --- a/test/util/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -189,7 +189,7 @@ ], "vout": [ { - "value": 1.3782, + "value": 1.37820000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index 6769ed79ff..5b69d0cd86 100644 --- a/test/util/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -198,7 +198,7 @@ ], "vout": [ { - "value": 1.3782, + "value": 1.37820000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index 82b64df075..cf1ebcdf38 100644 --- a/test/util/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -198,7 +198,7 @@ ], "vout": [ { - "value": 1.3782, + "value": 1.37820000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/txcreate1.json b/test/util/data/txcreate1.json index 36741044c9..edb091f946 100644 --- a/test/util/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -36,7 +36,7 @@ ], "vout": [ { - "value": 0.18, + "value": 0.18000000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", @@ -49,7 +49,7 @@ } }, { - "value": 4.00, + "value": 4.00000000, "n": 1, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 f2d4db28cad6502226ee484ae24505c2885cb12d OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/txcreate2.json b/test/util/data/txcreate2.json index 23fe7ace67..cca00f752b 100644 --- a/test/util/data/txcreate2.json +++ b/test/util/data/txcreate2.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "", diff --git a/test/util/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index e65a1859eb..e66a6bb9a5 100644 --- a/test/util/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -18,7 +18,7 @@ ], "vout": [ { - "value": 0.18, + "value": 0.18000000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", @@ -31,7 +31,7 @@ } }, { - "value": 4.00, + "value": 4.00000000, "n": 1, "scriptPubKey": { "asm": "OP_RETURN 54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", diff --git a/test/util/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index 8f1544e1c0..0f8edcafdd 100644 --- a/test/util/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -18,7 +18,7 @@ ], "vout": [ { - "value": 0.18, + "value": 0.18000000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", @@ -31,7 +31,7 @@ } }, { - "value": 0.00, + "value": 0.00000000, "n": 1, "scriptPubKey": { "asm": "OP_RETURN 54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", diff --git a/test/util/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json index e52401f418..4b5a7cab4a 100644 --- a/test/util/data/txcreatedata_seq0.json +++ b/test/util/data/txcreatedata_seq0.json @@ -18,7 +18,7 @@ ], "vout": [ { - "value": 0.18, + "value": 0.18000000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index 093ff4a56b..771ff1bb10 100644 --- a/test/util/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -27,7 +27,7 @@ ], "vout": [ { - "value": 0.18, + "value": 0.18000000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", diff --git a/test/util/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index 0cc530836a..7c814dad83 100644 --- a/test/util/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 1.00, + "value": 1.00000000, "n": 0, "scriptPubKey": { "asm": "2 02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397 021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d 02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485 3 OP_CHECKMULTISIG", diff --git a/test/util/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json index 8ad2ffdc65..7d94ce7396 100644 --- a/test/util/data/txcreatemultisig2.json +++ b/test/util/data/txcreatemultisig2.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 1.00, + "value": 1.00000000, "n": 0, "scriptPubKey": { "asm": "OP_HASH160 1c6fbaf46d64221e80cbae182c33ddf81b9294ac OP_EQUAL", diff --git a/test/util/data/txcreatemultisig3.json b/test/util/data/txcreatemultisig3.json index 086bf44b8a..06e093e224 100644 --- a/test/util/data/txcreatemultisig3.json +++ b/test/util/data/txcreatemultisig3.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 1.00, + "value": 1.00000000, "n": 0, "scriptPubKey": { "asm": "0 e15a86a23178f433d514dbbce042e87d72662b8b5edcacfd2e37ab7a2d135f05", diff --git a/test/util/data/txcreatemultisig4.json b/test/util/data/txcreatemultisig4.json index d23ccc045e..9a5d2f4a06 100644 --- a/test/util/data/txcreatemultisig4.json +++ b/test/util/data/txcreatemultisig4.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 1.00, + "value": 1.00000000, "n": 0, "scriptPubKey": { "asm": "OP_HASH160 6edf12858999f0dae74f9c692e6694ee3621b2ac OP_EQUAL", diff --git a/test/util/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json index f10aaecf7a..2704ed7673 100644 --- a/test/util/data/txcreateoutpubkey1.json +++ b/test/util/data/txcreateoutpubkey1.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397 OP_CHECKSIG", diff --git a/test/util/data/txcreateoutpubkey2.json b/test/util/data/txcreateoutpubkey2.json index 5a473b76c3..5144722230 100644 --- a/test/util/data/txcreateoutpubkey2.json +++ b/test/util/data/txcreateoutpubkey2.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "0 a2516e770582864a6a56ed21a102044e388c62e3", diff --git a/test/util/data/txcreateoutpubkey3.json b/test/util/data/txcreateoutpubkey3.json index b8389b8f7e..0a5d489e15 100644 --- a/test/util/data/txcreateoutpubkey3.json +++ b/test/util/data/txcreateoutpubkey3.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "OP_HASH160 a5ab14c9804d0d8bf02f1aea4e82780733ad0a83 OP_EQUAL", diff --git a/test/util/data/txcreatescript1.json b/test/util/data/txcreatescript1.json index 823168e9fb..5072452fed 100644 --- a/test/util/data/txcreatescript1.json +++ b/test/util/data/txcreatescript1.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "OP_DROP", diff --git a/test/util/data/txcreatescript2.json b/test/util/data/txcreatescript2.json index d4c7e10c78..94b669ffb6 100644 --- a/test/util/data/txcreatescript2.json +++ b/test/util/data/txcreatescript2.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "OP_HASH160 71ed53322d470bb96657deb786b94f97dd46fb15 OP_EQUAL", diff --git a/test/util/data/txcreatescript3.json b/test/util/data/txcreatescript3.json index 001e69511f..980da2fb31 100644 --- a/test/util/data/txcreatescript3.json +++ b/test/util/data/txcreatescript3.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "0 0bfe935e70c321c7ca3afc75ce0d0ca2f98b5422e008bb31c00c6d7f1f1c0ad6", diff --git a/test/util/data/txcreatescript4.json b/test/util/data/txcreatescript4.json index 20094bcd44..eecdf858b7 100644 --- a/test/util/data/txcreatescript4.json +++ b/test/util/data/txcreatescript4.json @@ -9,7 +9,7 @@ ], "vout": [ { - "value": 0.00, + "value": 0.00000000, "n": 0, "scriptPubKey": { "asm": "OP_HASH160 6a2c482f4985f57e702f325816c90e3723ca81ae OP_EQUAL", diff --git a/test/util/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json index 519d3ab066..92a3f76a07 100644 --- a/test/util/data/txcreatesignv1.json +++ b/test/util/data/txcreatesignv1.json @@ -18,7 +18,7 @@ ], "vout": [ { - "value": 0.001, + "value": 0.00100000, "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 5834479edbbe0539b31ffd3a8f8ebadc2165ed01 OP_EQUALVERIFY OP_CHECKSIG", |