diff options
Diffstat (limited to 'qa')
-rw-r--r-- | qa/README.md | 57 | ||||
-rwxr-xr-x | qa/pull-tester/rpc-tests.py | 185 | ||||
-rw-r--r-- | qa/rpc-tests/README.md | 49 | ||||
-rwxr-xr-x | qa/rpc-tests/getblocktemplate_longpoll.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/maxuploadtarget.py | 40 | ||||
-rwxr-xr-x | qa/rpc-tests/mempool_packages.py | 19 | ||||
-rwxr-xr-x | qa/rpc-tests/rpcbind_test.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/script_test.py | 259 | ||||
-rwxr-xr-x | qa/rpc-tests/smartfees.py | 52 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/authproxy.py | 10 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/coverage.py | 101 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/test_framework.py | 28 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/util.py | 62 |
13 files changed, 471 insertions, 395 deletions
diff --git a/qa/README.md b/qa/README.md new file mode 100644 index 0000000000..758d1f47e5 --- /dev/null +++ b/qa/README.md @@ -0,0 +1,57 @@ +The [pull-tester](/qa/pull-tester/) folder contains a script to call +multiple tests from the [rpc-tests](/qa/rpc-tests/) folder. + +Every pull request to the bitcoin repository is built and run through +the regression test suite. You can also run all or only individual +tests locally. + +Running tests +============= + +You can run any single test by calling `qa/pull-tester/rpc-tests.py <testname>`. + +Or you can run any combination of tests by calling `qa/pull-tester/rpc-tests.py <testname1> <testname2> <testname3> ...` + +Run the regression test suite with `qa/pull-tester/rpc-tests.py` + +Run all possible tests with `qa/pull-tester/rpc-tests.py -extended` + +Possible options: + +``` + -h, --help show this help message and exit + --nocleanup Leave bitcoinds and test.* datadir on exit or error + --noshutdown Don't stop bitcoinds after the test execution + --srcdir=SRCDIR Source directory containing bitcoind/bitcoin-cli + (default: ../../src) + --tmpdir=TMPDIR Root directory for datadirs + --tracerpc Print out all RPC calls as they are made + --coveragedir=COVERAGEDIR + Write tested RPC commands into this directory +``` + +If you set the environment variable `PYTHON_DEBUG=1` you will get some debug +output (example: `PYTHON_DEBUG=1 qa/pull-tester/rpc-tests.py wallet`). + +A 200-block -regtest blockchain and wallets for four nodes +is created the first time a regression test is run and +is stored in the cache/ directory. Each node has 25 mature +blocks (25*50=1250 BTC) in its wallet. + +After the first run, the cache/ blockchain and wallets are +copied into a temporary directory and used as the initial +test state. + +If you get into a bad state, you should be able +to recover with: + +```bash +rm -rf cache +killall bitcoind +``` + +Writing tests +============= +You are encouraged to write tests for new or existing features. +Further information about the test framework and individual rpc +tests is found in [qa/rpc-tests](/qa/rpc-tests). diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 86a416edc4..3d156a2e7b 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -3,16 +3,32 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Run Regression Test Suite -# +""" +Run Regression Test Suite + +This module calls down into individual test cases via subprocess. It will +forward all unrecognized arguments onto the individual test scripts, other +than: + + - `-extended`: run the "extended" test suite in addition to the basic one. + - `-win`: signal that this is running in a Windows environment, and we + should run the tests. + - `--coverage`: this generates a basic coverage report for the RPC + interface. + +For a description of arguments recognized by test scripts, see +`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. + +""" import os +import shutil import sys import subprocess +import tempfile import re + from tests_config import * -from sets import Set #If imported values are not defined then set to zero (or disabled) if not vars().has_key('ENABLE_WALLET'): @@ -24,15 +40,20 @@ if not vars().has_key('ENABLE_UTILS'): if not vars().has_key('ENABLE_ZMQ'): ENABLE_ZMQ=0 +ENABLE_COVERAGE=0 + #Create a set to store arguments and create the passOn string -opts = Set() +opts = set() passOn = "" p = re.compile("^--") -for i in range(1,len(sys.argv)): - if (p.match(sys.argv[i]) or sys.argv[i] == "-h"): - passOn += " " + sys.argv[i] + +for arg in sys.argv[1:]: + if arg == '--coverage': + ENABLE_COVERAGE = 1 + elif (p.match(arg) or arg == "-h"): + passOn += " " + arg else: - opts.add(sys.argv[i]) + opts.add(arg) #Set env vars buildDir = BUILDDIR @@ -48,6 +69,7 @@ if EXEEXT == ".exe" and "-win" not in opts: testScripts = [ 'wallet.py', 'listtransactions.py', + 'receivedby.py', 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', 'txn_clone.py', @@ -83,9 +105,7 @@ testScriptsExt = [ 'forknotify.py', 'invalidateblock.py', 'keypool.py', - 'receivedby.py', # 'rpcbind_test.py', #temporary, bug in libevent, see #6655 -# 'script_test.py', #used for manual comparison of 2 binaries 'smartfees.py', 'maxblocksinflight.py', 'invalidblockrequest.py', @@ -99,24 +119,125 @@ testScriptsExt = [ if ENABLE_ZMQ == 1: testScripts.append('zmq_test.py') -if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): - rpcTestDir = buildDir + '/qa/rpc-tests/' - #Run Tests - for i in range(len(testScripts)): - if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts - or testScripts[i] in opts or re.sub(".py$", "", testScripts[i]) in opts ): - print "Running testscript " + testScripts[i] + "..." - subprocess.check_call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) - #exit if help is called so we print just one set of instructions - p = re.compile(" -h| --help") - if p.match(passOn): - sys.exit(0) - - #Run Extended Tests - for i in range(len(testScriptsExt)): - if ('-extended' in opts or testScriptsExt[i] in opts - or re.sub(".py$", "", testScriptsExt[i]) in opts): - print "Running 2nd level testscript " + testScriptsExt[i] + "..." - subprocess.check_call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) -else: - print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" + +def runtests(): + coverage = None + + if ENABLE_COVERAGE: + coverage = RPCCoverage() + print("Initializing coverage directory at %s" % coverage.dir) + + if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): + rpcTestDir = buildDir + '/qa/rpc-tests/' + run_extended = '-extended' in opts + cov_flag = coverage.flag if coverage else '' + flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn) + + #Run Tests + for i in range(len(testScripts)): + if (len(opts) == 0 + or (len(opts) == 1 and "-win" in opts ) + or run_extended + or testScripts[i] in opts + or re.sub(".py$", "", testScripts[i]) in opts ): + print("Running testscript " + testScripts[i] + "...") + + subprocess.check_call( + rpcTestDir + testScripts[i] + flags, shell=True) + + # exit if help is called so we print just one set of + # instructions + p = re.compile(" -h| --help") + if p.match(passOn): + sys.exit(0) + + # Run Extended Tests + for i in range(len(testScriptsExt)): + if (run_extended or testScriptsExt[i] in opts + or re.sub(".py$", "", testScriptsExt[i]) in opts): + print( + "Running 2nd level testscript " + + testScriptsExt[i] + "...") + + subprocess.check_call( + rpcTestDir + testScriptsExt[i] + flags, shell=True) + + if coverage: + coverage.report_rpc_coverage() + + print("Cleaning up coverage data") + coverage.cleanup() + + else: + print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" + + +class RPCCoverage(object): + """ + Coverage reporting utilities for pull-tester. + + Coverage calculation works by having each test script subprocess write + coverage files into a particular directory. These files contain the RPC + commands invoked during testing, as well as a complete listing of RPC + commands per `bitcoin-cli help` (`rpc_interface.txt`). + + After all tests complete, the commands run are combined and diff'd against + the complete list to calculate uncovered RPC commands. + + See also: qa/rpc-tests/test_framework/coverage.py + + """ + def __init__(self): + self.dir = tempfile.mkdtemp(prefix="coverage") + self.flag = '--coveragedir %s' % self.dir + + def report_rpc_coverage(self): + """ + Print out RPC commands that were unexercised by tests. + + """ + uncovered = self._get_uncovered_rpc_commands() + + if uncovered: + print("Uncovered RPC commands:") + print("".join((" - %s\n" % i) for i in sorted(uncovered))) + else: + print("All RPC commands covered.") + + def cleanup(self): + return shutil.rmtree(self.dir) + + def _get_uncovered_rpc_commands(self): + """ + Return a set of currently untested RPC commands. + + """ + # This is shared from `qa/rpc-tests/test-framework/coverage.py` + REFERENCE_FILENAME = 'rpc_interface.txt' + COVERAGE_FILE_PREFIX = 'coverage.' + + coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) + coverage_filenames = set() + all_cmds = set() + covered_cmds = set() + + if not os.path.isfile(coverage_ref_filename): + raise RuntimeError("No coverage reference found") + + with open(coverage_ref_filename, 'r') as f: + all_cmds.update([i.strip() for i in f.readlines()]) + + for root, dirs, files in os.walk(self.dir): + for filename in files: + if filename.startswith(COVERAGE_FILE_PREFIX): + coverage_filenames.add(os.path.join(root, filename)) + + for filename in coverage_filenames: + with open(filename, 'r') as f: + covered_cmds.update([i.strip() for i in f.readlines()]) + + return all_cmds - covered_cmds + + +if __name__ == '__main__': + runtests() diff --git a/qa/rpc-tests/README.md b/qa/rpc-tests/README.md index d2db00362f..898931936b 100644 --- a/qa/rpc-tests/README.md +++ b/qa/rpc-tests/README.md @@ -1,10 +1,8 @@ Regression tests ================ -### [python-bitcoinrpc](https://github.com/jgarzik/python-bitcoinrpc) -Git subtree of [https://github.com/jgarzik/python-bitcoinrpc](https://github.com/jgarzik/python-bitcoinrpc). -Changes to python-bitcoinrpc should be made upstream, and then -pulled here using git subtree. +### [test_framework/authproxy.py](test_framework/authproxy.py) +Taken from the [python-bitcoinrpc repository](https://github.com/jgarzik/python-bitcoinrpc). ### [test_framework/test_framework.py](test_framework/test_framework.py) Base class for new regression tests. @@ -33,49 +31,6 @@ Helpers for script.py ### [test_framework/blocktools.py](test_framework/blocktools.py) Helper functions for creating blocks and transactions. - -Notes -===== - -You can run any single test by calling `qa/pull-tester/rpc-tests.py <testname>`. - -Or you can run any combination of tests by calling `qa/pull-tester/rpc-tests.py <testname1> <testname2> <testname3> ...` - -Run the regression test suite with `qa/pull-tester/rpc-tests.py` - -Run all possible tests with `qa/pull-tester/rpc-tests.py -extended` - -Possible options: - -``` --h, --help show this help message and exit - --nocleanup Leave bitcoinds and test.* datadir on exit or error - --noshutdown Don't stop bitcoinds after the test execution - --srcdir=SRCDIR Source directory containing bitcoind/bitcoin-cli (default: - ../../src) - --tmpdir=TMPDIR Root directory for datadirs - --tracerpc Print out all RPC calls as they are made -``` - -If you set the environment variable `PYTHON_DEBUG=1` you will get some debug output (example: `PYTHON_DEBUG=1 qa/pull-tester/rpc-tests.py wallet`). - -A 200-block -regtest blockchain and wallets for four nodes -is created the first time a regression test is run and -is stored in the cache/ directory. Each node has 25 mature -blocks (25*50=1250 BTC) in its wallet. - -After the first run, the cache/ blockchain and wallets are -copied into a temporary directory and used as the initial -test state. - -If you get into a bad state, you should be able -to recover with: - -```bash -rm -rf cache -killall bitcoind -``` - P2P test design notes --------------------- diff --git a/qa/rpc-tests/getblocktemplate_longpoll.py b/qa/rpc-tests/getblocktemplate_longpoll.py index aab4562422..1ddff8a298 100755 --- a/qa/rpc-tests/getblocktemplate_longpoll.py +++ b/qa/rpc-tests/getblocktemplate_longpoll.py @@ -38,7 +38,7 @@ class LongpollThread(threading.Thread): self.longpollid = templat['longpollid'] # create a new connection to the node, we can't use the same # connection from two threads - self.node = AuthServiceProxy(node.url, timeout=600) + self.node = get_rpc_proxy(node.url, 1, timeout=600) def run(self): self.node.getblocktemplate({'longpollid':self.longpollid}) diff --git a/qa/rpc-tests/maxuploadtarget.py b/qa/rpc-tests/maxuploadtarget.py index 67c4a50985..e714465db1 100755 --- a/qa/rpc-tests/maxuploadtarget.py +++ b/qa/rpc-tests/maxuploadtarget.py @@ -192,9 +192,10 @@ class MaxUploadTest(BitcoinTestFramework): getdata_request.inv.append(CInv(2, big_old_block)) max_bytes_per_day = 200*1024*1024 - max_bytes_available = max_bytes_per_day - 144*1000000 + daily_buffer = 144 * 1000000 + max_bytes_available = max_bytes_per_day - daily_buffer success_count = max_bytes_available / old_block_size - + # 144MB will be reserved for relaying new blocks, so expect this to # succeed for ~70 tries. for i in xrange(success_count): @@ -227,7 +228,7 @@ class MaxUploadTest(BitcoinTestFramework): test_nodes[1].send_message(getdata_request) test_nodes[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) - + print "Peer 1 disconnected after trying to download old block" print "Advancing system time on node to clear counters..." @@ -244,5 +245,38 @@ class MaxUploadTest(BitcoinTestFramework): [c.disconnect_node() for c in connections] + #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 + print "Restarting nodes with -whitelist=127.0.0.1" + stop_node(self.nodes[0], 0) + self.nodes[0] = start_node(0, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) + + #recreate/reconnect 3 test nodes + test_nodes = [] + connections = [] + + for i in xrange(3): + test_nodes.append(TestNode()) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) + test_nodes[i].add_connection(connections[i]) + + NetworkThread().start() # Start up network handling in another thread + [x.wait_for_verack() for x in test_nodes] + + #retrieve 20 blocks which should be enough to break the 1MB limit + getdata_request.inv = [CInv(2, big_new_block)] + for i in xrange(20): + test_nodes[1].send_message(getdata_request) + test_nodes[1].sync_with_ping() + assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + + getdata_request.inv = [CInv(2, big_old_block)] + test_nodes[1].send_message(getdata_request) + test_nodes[1].wait_for_disconnect() + assert_equal(len(self.nodes[0].getpeerinfo()), 3) #node is still connected because of the whitelist + + print "Peer 1 still connected after trying to download old block (whitelisted)" + + [c.disconnect_node() for c in connections] + if __name__ == '__main__': MaxUploadTest().main() diff --git a/qa/rpc-tests/mempool_packages.py b/qa/rpc-tests/mempool_packages.py index 6bc6e43f0b..746c26ff5e 100755 --- a/qa/rpc-tests/mempool_packages.py +++ b/qa/rpc-tests/mempool_packages.py @@ -11,6 +11,9 @@ from test_framework.util import * def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) +MAX_ANCESTORS = 25 +MAX_DESCENDANTS = 25 + class MempoolPackagesTest(BitcoinTestFramework): def setup_network(self): @@ -45,17 +48,17 @@ class MempoolPackagesTest(BitcoinTestFramework): value = utxo[0]['amount'] fee = Decimal("0.0001") - # 100 transactions off a confirmed tx should be fine + # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] - for i in xrange(100): + for i in xrange(MAX_ANCESTORS): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, 0, value, fee, 1) value = sent_value chain.append(txid) - # Check mempool has 100 transactions in it, and descendant + # Check mempool has MAX_ANCESTORS transactions in it, and descendant # count and fees should look correct mempool = self.nodes[0].getrawmempool(True) - assert_equal(len(mempool), 100) + assert_equal(len(mempool), MAX_ANCESTORS) descendant_count = 1 descendant_fees = 0 descendant_size = 0 @@ -91,18 +94,18 @@ class MempoolPackagesTest(BitcoinTestFramework): for i in xrange(10): transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) - for i in xrange(1000): + for i in xrange(MAX_DESCENDANTS): utxo = transaction_package.pop(0) try: (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) for j in xrange(10): transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) - if i == 998: + if i == MAX_DESCENDANTS - 2: mempool = self.nodes[0].getrawmempool(True) - assert_equal(mempool[parent_transaction]['descendantcount'], 1000) + assert_equal(mempool[parent_transaction]['descendantcount'], MAX_DESCENDANTS) except JSONRPCException as e: print e.error['message'] - assert_equal(i, 999) + assert_equal(i, MAX_DESCENDANTS - 1) print "tx that would create too large descendant package successfully rejected" # TODO: check that node1's mempool is as expected diff --git a/qa/rpc-tests/rpcbind_test.py b/qa/rpc-tests/rpcbind_test.py index 04110c2831..7a9da66787 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/qa/rpc-tests/rpcbind_test.py @@ -47,7 +47,7 @@ def run_allowip_test(tmpdir, allow_ips, rpchost, rpcport): try: # connect to node through non-loopback interface url = "http://rt:rt@%s:%d" % (rpchost, rpcport,) - node = AuthServiceProxy(url) + node = get_rpc_proxy(url, 1) node.getinfo() finally: node = None # make sure connection will be garbage collected and closed diff --git a/qa/rpc-tests/script_test.py b/qa/rpc-tests/script_test.py deleted file mode 100755 index afc44b51b5..0000000000 --- a/qa/rpc-tests/script_test.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python2 -# -# Distributed under the MIT/X11 software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -''' -Test notes: -This test uses the script_valid and script_invalid tests from the unittest -framework to do end-to-end testing where we compare that two nodes agree on -whether blocks containing a given test script are valid. - -We generally ignore the script flags associated with each test (since we lack -the precision to test each script using those flags in this framework), but -for tests with SCRIPT_VERIFY_P2SH, we can use a block time after the BIP16 -switchover date to try to test with that flag enabled (and for tests without -that flag, we use a block time before the switchover date). - -NOTE: This test is very slow and may take more than 40 minutes to run. -''' - -from test_framework.test_framework import ComparisonTestFramework -from test_framework.util import * -from test_framework.comptool import TestInstance, TestManager -from test_framework.mininode import * -from test_framework.blocktools import * -from test_framework.script import * -import logging -import copy -import json - -script_valid_file = "../../src/test/data/script_valid.json" -script_invalid_file = "../../src/test/data/script_invalid.json" - -# Pass in a set of json files to open. -class ScriptTestFile(object): - - def __init__(self, files): - self.files = files - self.index = -1 - self.data = [] - - def load_files(self): - for f in self.files: - self.data.extend(json.loads(open(os.path.dirname(os.path.abspath(__file__))+"/"+f).read())) - - # Skip over records that are not long enough to be tests - def get_records(self): - while (self.index < len(self.data)): - if len(self.data[self.index]) >= 3: - yield self.data[self.index] - self.index += 1 - - -# Helper for parsing the flags specified in the .json files -SCRIPT_VERIFY_NONE = 0 -SCRIPT_VERIFY_P2SH = 1 -SCRIPT_VERIFY_STRICTENC = 1 << 1 -SCRIPT_VERIFY_DERSIG = 1 << 2 -SCRIPT_VERIFY_LOW_S = 1 << 3 -SCRIPT_VERIFY_NULLDUMMY = 1 << 4 -SCRIPT_VERIFY_SIGPUSHONLY = 1 << 5 -SCRIPT_VERIFY_MINIMALDATA = 1 << 6 -SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS = 1 << 7 -SCRIPT_VERIFY_CLEANSTACK = 1 << 8 - -flag_map = { - "": SCRIPT_VERIFY_NONE, - "NONE": SCRIPT_VERIFY_NONE, - "P2SH": SCRIPT_VERIFY_P2SH, - "STRICTENC": SCRIPT_VERIFY_STRICTENC, - "DERSIG": SCRIPT_VERIFY_DERSIG, - "LOW_S": SCRIPT_VERIFY_LOW_S, - "NULLDUMMY": SCRIPT_VERIFY_NULLDUMMY, - "SIGPUSHONLY": SCRIPT_VERIFY_SIGPUSHONLY, - "MINIMALDATA": SCRIPT_VERIFY_MINIMALDATA, - "DISCOURAGE_UPGRADABLE_NOPS": SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS, - "CLEANSTACK": SCRIPT_VERIFY_CLEANSTACK, -} - -def ParseScriptFlags(flag_string): - flags = 0 - for x in flag_string.split(","): - if x in flag_map: - flags |= flag_map[x] - else: - print "Error: unrecognized script flag: ", x - return flags - -''' -Given a string that is a scriptsig or scriptpubkey from the .json files above, -convert it to a CScript() -''' -# Replicates behavior from core_read.cpp -def ParseScript(json_script): - script = json_script.split(" ") - parsed_script = CScript() - for x in script: - if len(x) == 0: - # Empty string, ignore. - pass - elif x.isdigit() or (len(x) >= 1 and x[0] == "-" and x[1:].isdigit()): - # Number - n = int(x, 0) - if (n == -1) or (n >= 1 and n <= 16): - parsed_script = CScript(bytes(parsed_script) + bytes(CScript([n]))) - else: - parsed_script += CScriptNum(int(x, 0)) - elif x.startswith("0x"): - # Raw hex data, inserted NOT pushed onto stack: - for i in xrange(2, len(x), 2): - parsed_script = CScript(bytes(parsed_script) + bytes(chr(int(x[i:i+2],16)))) - elif x.startswith("'") and x.endswith("'") and len(x) >= 2: - # Single-quoted string, pushed as data. - parsed_script += CScript([x[1:-1]]) - else: - # opcode, e.g. OP_ADD or ADD: - tryopname = "OP_" + x - if tryopname in OPCODES_BY_NAME: - parsed_script += CScriptOp(OPCODES_BY_NAME["OP_" + x]) - else: - print "ParseScript: error parsing '%s'" % x - return "" - return parsed_script - -class TestBuilder(object): - def create_credit_tx(self, scriptPubKey, height): - # self.tx1 is a coinbase transaction, modeled after the one created by script_tests.cpp - # This allows us to reuse signatures created in the unit test framework. - self.tx1 = create_coinbase(height) # this has a bip34 scriptsig, - self.tx1.vin[0].scriptSig = CScript([0, 0]) # but this matches the unit tests - self.tx1.vout[0].nValue = 0 - self.tx1.vout[0].scriptPubKey = scriptPubKey - self.tx1.rehash() - def create_spend_tx(self, scriptSig): - self.tx2 = create_transaction(self.tx1, 0, CScript(), 0) - self.tx2.vin[0].scriptSig = scriptSig - self.tx2.vout[0].scriptPubKey = CScript() - self.tx2.rehash() - def rehash(self): - self.tx1.rehash() - self.tx2.rehash() - -# This test uses the (default) two nodes provided by ComparisonTestFramework, -# specified on the command line with --testbinary and --refbinary. -# See comptool.py -class ScriptTest(ComparisonTestFramework): - - def run_test(self): - # Set up the comparison tool TestManager - test = TestManager(self, self.options.tmpdir) - test.add_all_connections(self.nodes) - - # Load scripts - self.scripts = ScriptTestFile([script_valid_file, script_invalid_file]) - self.scripts.load_files() - - # Some variables we re-use between test instances (to build blocks) - self.tip = None - self.block_time = None - - NetworkThread().start() # Start up network handling in another thread - test.run() - - def generate_test_instance(self, pubkeystring, scriptsigstring): - scriptpubkey = ParseScript(pubkeystring) - scriptsig = ParseScript(scriptsigstring) - - test = TestInstance(sync_every_block=False) - test_build = TestBuilder() - test_build.create_credit_tx(scriptpubkey, self.height) - test_build.create_spend_tx(scriptsig) - test_build.rehash() - - block = create_block(self.tip, test_build.tx1, self.block_time) - self.block_time += 1 - block.solve() - self.tip = block.sha256 - self.height += 1 - test.blocks_and_transactions = [[block, True]] - - for i in xrange(100): - block = create_block(self.tip, create_coinbase(self.height), self.block_time) - self.block_time += 1 - block.solve() - self.tip = block.sha256 - self.height += 1 - test.blocks_and_transactions.append([block, True]) - - block = create_block(self.tip, create_coinbase(self.height), self.block_time) - self.block_time += 1 - block.vtx.append(test_build.tx2) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() - block.solve() - test.blocks_and_transactions.append([block, None]) - return test - - # This generates the tests for TestManager. - def get_tests(self): - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) - self.block_time = 1333230000 # before the BIP16 switchover - self.height = 1 - - ''' - Create a new block with an anyone-can-spend coinbase - ''' - block = create_block(self.tip, create_coinbase(self.height), self.block_time) - self.block_time += 1 - block.solve() - self.tip = block.sha256 - self.height += 1 - yield TestInstance(objects=[[block, True]]) - - ''' - Build out to 100 blocks total, maturing the coinbase. - ''' - test = TestInstance(objects=[], sync_every_block=False, sync_every_tx=False) - for i in xrange(100): - b = create_block(self.tip, create_coinbase(self.height), self.block_time) - b.solve() - test.blocks_and_transactions.append([b, True]) - self.tip = b.sha256 - self.block_time += 1 - self.height += 1 - yield test - - ''' Iterate through script tests. ''' - counter = 0 - for script_test in self.scripts.get_records(): - ''' Reset the blockchain to genesis block + 100 blocks. ''' - if self.nodes[0].getblockcount() > 101: - self.nodes[0].invalidateblock(self.nodes[0].getblockhash(102)) - self.nodes[1].invalidateblock(self.nodes[1].getblockhash(102)) - - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) - self.height = 102 - - [scriptsig, scriptpubkey, flags] = script_test[0:3] - flags = ParseScriptFlags(flags) - - # We can use block time to determine whether the nodes should be - # enforcing BIP16. - # - # We intentionally let the block time grow by 1 each time. - # This forces the block hashes to differ between tests, so that - # a call to invalidateblock doesn't interfere with a later test. - if (flags & SCRIPT_VERIFY_P2SH): - self.block_time = 1333238400 + counter # Advance to enforcing BIP16 - else: - self.block_time = 1333230000 + counter # Before the BIP16 switchover - - print "Script test: [%s]" % script_test - - yield self.generate_test_instance(scriptpubkey, scriptsig) - counter += 1 - -if __name__ == '__main__': - ScriptTest().main() diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index c15c5fda09..ecfffc1b45 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -120,15 +120,26 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): last_e = e valid_estimate = False invalid_estimates = 0 - for e in all_estimates: + for i,e in enumerate(all_estimates): # estimate is for i+1 if e >= 0: valid_estimate = True + # estimatesmartfee should return the same result + assert_equal(node.estimatesmartfee(i+1)["feerate"], e) + else: invalid_estimates += 1 - # Once we're at a high enough confirmation count that we can give an estimate - # We should have estimates for all higher confirmation counts - if valid_estimate and e < 0: - raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate") + + # estimatesmartfee should still be valid + approx_estimate = node.estimatesmartfee(i+1)["feerate"] + answer_found = node.estimatesmartfee(i+1)["blocks"] + assert(approx_estimate > 0) + assert(answer_found > i+1) + + # Once we're at a high enough confirmation count that we can give an estimate + # We should have estimates for all higher confirmation counts + if valid_estimate: + raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate") + # Check on the expected number of different confirmation counts # that we might not have valid estimates for if invalid_estimates > max_invalid: @@ -184,13 +195,13 @@ class EstimateFeeTest(BitcoinTestFramework): # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, # (17k is room enough for 110 or so transactions) self.nodes.append(start_node(1, self.options.tmpdir, - ["-blockprioritysize=1500", "-blockmaxsize=18000", + ["-blockprioritysize=1500", "-blockmaxsize=17000", "-maxorphantx=1000", "-relaypriority=0", "-debug=estimatefee"])) connect_nodes(self.nodes[1], 0) # Node2 is a stingy miner, that - # produces too small blocks (room for only 70 or so transactions) - node2args = ["-blockprioritysize=0", "-blockmaxsize=12000", "-maxorphantx=1000", "-relaypriority=0"] + # produces too small blocks (room for only 55 or so transactions) + node2args = ["-blockprioritysize=0", "-blockmaxsize=8000", "-maxorphantx=1000", "-relaypriority=0"] self.nodes.append(start_node(2, self.options.tmpdir, node2args)) connect_nodes(self.nodes[0], 2) @@ -229,22 +240,19 @@ class EstimateFeeTest(BitcoinTestFramework): self.fees_per_kb = [] self.memutxo = [] self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting - print("Checking estimates for 1/2/3/6/15/25 blocks") - print("Creating transactions and mining them with a huge block size") - # Create transactions and mine 20 big blocks with node 0 such that the mempool is always emptied - self.transact_and_mine(30, self.nodes[0]) - check_estimates(self.nodes[1], self.fees_per_kb, 1) + print("Will output estimates for 1/2/3/6/15/25 blocks") - print("Creating transactions and mining them with a block size that can't keep up") - # Create transactions and mine 30 small blocks with node 2, but create txs faster than we can mine - self.transact_and_mine(20, self.nodes[2]) - check_estimates(self.nodes[1], self.fees_per_kb, 3) + for i in xrange(2): + print("Creating transactions and mining them with a block size that can't keep up") + # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine + self.transact_and_mine(10, self.nodes[2]) + check_estimates(self.nodes[1], self.fees_per_kb, 14) - print("Creating transactions and mining them at a block size that is just big enough") - # Generate transactions while mining 40 more blocks, this time with node1 - # which mines blocks with capacity just above the rate that transactions are being created - self.transact_and_mine(40, self.nodes[1]) - check_estimates(self.nodes[1], self.fees_per_kb, 2) + print("Creating transactions and mining them at a block size that is just big enough") + # Generate transactions while mining 10 more blocks, this time with node1 + # which mines blocks with capacity just above the rate that transactions are being created + self.transact_and_mine(10, self.nodes[1]) + check_estimates(self.nodes[1], self.fees_per_kb, 2) # Finish by mining a normal-sized block: while len(self.nodes[1].getrawmempool()) > 0: diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 33014dc139..fba469a0dd 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -69,7 +69,7 @@ class AuthServiceProxy(object): def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None): self.__service_url = service_url - self.__service_name = service_name + self._service_name = service_name self.__url = urlparse.urlparse(service_url) if self.__url.port is None: port = 80 @@ -102,8 +102,8 @@ class AuthServiceProxy(object): if name.startswith('__') and name.endswith('__'): # Python internal stuff raise AttributeError - if self.__service_name is not None: - name = "%s.%s" % (self.__service_name, name) + if self._service_name is not None: + name = "%s.%s" % (self._service_name, name) return AuthServiceProxy(self.__service_url, name, connection=self.__conn) def _request(self, method, path, postdata): @@ -129,10 +129,10 @@ class AuthServiceProxy(object): def __call__(self, *args): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self.__service_name, + log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name, json.dumps(args, default=EncodeDecimal))) postdata = json.dumps({'version': '1.1', - 'method': self.__service_name, + 'method': self._service_name, 'params': args, 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal) response = self._request('POST', self.__url.path, postdata) diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py new file mode 100644 index 0000000000..50f066a850 --- /dev/null +++ b/qa/rpc-tests/test_framework/coverage.py @@ -0,0 +1,101 @@ +""" +This module contains utilities for doing coverage analysis on the RPC +interface. + +It provides a way to track which RPC commands are exercised during +testing. + +""" +import os + + +REFERENCE_FILENAME = 'rpc_interface.txt' + + +class AuthServiceProxyWrapper(object): + """ + An object that wraps AuthServiceProxy to record specific RPC calls. + + """ + def __init__(self, auth_service_proxy_instance, coverage_logfile=None): + """ + Kwargs: + auth_service_proxy_instance (AuthServiceProxy): the instance + being wrapped. + coverage_logfile (str): if specified, write each service_name + out to a file when called. + + """ + self.auth_service_proxy_instance = auth_service_proxy_instance + self.coverage_logfile = coverage_logfile + + def __getattr__(self, *args, **kwargs): + return_val = self.auth_service_proxy_instance.__getattr__( + *args, **kwargs) + + return AuthServiceProxyWrapper(return_val, self.coverage_logfile) + + def __call__(self, *args, **kwargs): + """ + Delegates to AuthServiceProxy, then writes the particular RPC method + called to a file. + + """ + return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs) + rpc_method = self.auth_service_proxy_instance._service_name + + if self.coverage_logfile: + with open(self.coverage_logfile, 'a+') as f: + f.write("%s\n" % rpc_method) + + return return_val + + @property + def url(self): + return self.auth_service_proxy_instance.url + + +def get_filename(dirname, n_node): + """ + Get a filename unique to the test process ID and node. + + This file will contain a list of RPC commands covered. + """ + pid = str(os.getpid()) + return os.path.join( + dirname, "coverage.pid%s.node%s.txt" % (pid, str(n_node))) + + +def write_all_rpc_commands(dirname, node): + """ + Write out a list of all RPC functions available in `bitcoin-cli` for + coverage comparison. This will only happen once per coverage + directory. + + Args: + dirname (str): temporary test dir + node (AuthServiceProxy): client + + Returns: + bool. if the RPC interface file was written. + + """ + filename = os.path.join(dirname, REFERENCE_FILENAME) + + if os.path.isfile(filename): + return False + + help_output = node.help().split('\n') + commands = set() + + for line in help_output: + line = line.strip() + + # Ignore blanks and headers + if line and not line.startswith('='): + commands.add("%s\n" % line.split()[0]) + + with open(filename, 'w') as f: + f.writelines(list(commands)) + + return True diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 5671431f6e..ae2d91ab60 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -13,8 +13,20 @@ import shutil import tempfile import traceback +from .util import ( + initialize_chain, + assert_equal, + start_nodes, + connect_nodes_bi, + sync_blocks, + sync_mempools, + stop_nodes, + wait_bitcoinds, + enable_coverage, + check_json_precision, + initialize_chain_clean, +) from authproxy import AuthServiceProxy, JSONRPCException -from util import * class BitcoinTestFramework(object): @@ -96,6 +108,8 @@ class BitcoinTestFramework(object): help="Root directory for datadirs") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") + parser.add_option("--coveragedir", dest="coveragedir", + help="Write tested RPC commands into this directory") self.add_options(parser) (self.options, self.args) = parser.parse_args() @@ -103,6 +117,9 @@ class BitcoinTestFramework(object): import logging logging.basicConfig(level=logging.DEBUG) + if self.options.coveragedir: + enable_coverage(self.options.coveragedir) + os.environ['PATH'] = self.options.srcdir+":"+os.environ['PATH'] check_json_precision() @@ -173,7 +190,8 @@ class ComparisonTestFramework(BitcoinTestFramework): initialize_chain_clean(self.options.tmpdir, self.num_nodes) def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1']] * self.num_nodes, - binary=[self.options.testbinary] + - [self.options.refbinary]*(self.num_nodes-1)) + self.nodes = start_nodes( + self.num_nodes, self.options.tmpdir, + extra_args=[['-debug', '-whitelist=127.0.0.1']] * self.num_nodes, + binary=[self.options.testbinary] + + [self.options.refbinary]*(self.num_nodes-1)) diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 3759cc8162..30dd5de585 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -17,8 +17,43 @@ import subprocess import time import re -from authproxy import AuthServiceProxy, JSONRPCException -from util import * +from . import coverage +from .authproxy import AuthServiceProxy, JSONRPCException + +COVERAGE_DIR = None + + +def enable_coverage(dirname): + """Maintain a log of which RPC calls are made during testing.""" + global COVERAGE_DIR + COVERAGE_DIR = dirname + + +def get_rpc_proxy(url, node_number, timeout=None): + """ + Args: + url (str): URL of the RPC server to call + node_number (int): the node number (or id) that this calls to + + Kwargs: + timeout (int): HTTP timeout in seconds + + Returns: + AuthServiceProxy. convenience object for making RPC calls. + + """ + proxy_kwargs = {} + if timeout is not None: + proxy_kwargs['timeout'] = timeout + + proxy = AuthServiceProxy(url, **proxy_kwargs) + proxy.url = url # store URL on proxy for info + + coverage_logfile = coverage.get_filename( + COVERAGE_DIR, node_number) if COVERAGE_DIR else None + + return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) + def p2p_port(n): return 11000 + n + os.getpid()%999 @@ -79,13 +114,13 @@ def initialize_chain(test_dir): """ if (not os.path.isdir(os.path.join("cache","node0")) - or not os.path.isdir(os.path.join("cache","node1")) - or not os.path.isdir(os.path.join("cache","node2")) + or not os.path.isdir(os.path.join("cache","node1")) + or not os.path.isdir(os.path.join("cache","node2")) or not os.path.isdir(os.path.join("cache","node3"))): #find and delete old cache directories if any exist for i in range(4): - if os.path.isdir(os.path.join("cache","node"+str(i))): + if os.path.isdir(os.path.join("cache","node"+str(i))): shutil.rmtree(os.path.join("cache","node"+str(i))) devnull = open(os.devnull, "w") @@ -103,11 +138,13 @@ def initialize_chain(test_dir): if os.getenv("PYTHON_DEBUG", ""): print "initialize_chain: bitcoin-cli -rpcwait getblockcount completed" devnull.close() + rpcs = [] + for i in range(4): try: - url = "http://rt:rt@127.0.0.1:%d"%(rpc_port(i),) - rpcs.append(AuthServiceProxy(url)) + url = "http://rt:rt@127.0.0.1:%d" % (rpc_port(i),) + rpcs.append(get_rpc_proxy(url, i)) except: sys.stderr.write("Error connecting to "+url+"\n") sys.exit(1) @@ -190,11 +227,12 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= print "start_node: calling bitcoin-cli -rpcwait getblockcount returned" devnull.close() url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i)) - if timewait is not None: - proxy = AuthServiceProxy(url, timeout=timewait) - else: - proxy = AuthServiceProxy(url) - proxy.url = url # store URL on proxy for info + + proxy = get_rpc_proxy(url, i, timeout=timewait) + + if COVERAGE_DIR: + coverage.write_all_rpc_commands(COVERAGE_DIR, proxy) + return proxy def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None): |