diff options
Diffstat (limited to 'test/functional/test_framework')
| -rwxr-xr-x | test/functional/test_framework/comptool.py | 12 | ||||
| -rwxr-xr-x | test/functional/test_framework/mininode.py | 48 | ||||
| -rwxr-xr-x | test/functional/test_framework/test_framework.py | 269 | ||||
| -rwxr-xr-x | test/functional/test_framework/test_node.py | 190 | ||||
| -rw-r--r-- | test/functional/test_framework/util.py | 63 |
5 files changed, 391 insertions, 191 deletions
diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py index 9f062865a3..bfbc0c3b03 100755 --- a/test/functional/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -19,7 +19,7 @@ TestNode behaves as follows: from .mininode import * from .blockstore import BlockStore, TxStore -from .util import p2p_port +from .util import p2p_port, wait_until import logging @@ -189,7 +189,7 @@ class TestManager(object): def wait_for_disconnections(self): def disconnected(): return all(node.closed for node in self.test_nodes) - return wait_until(disconnected, timeout=10) + wait_until(disconnected, timeout=10, lock=mininode_lock) def wait_for_verack(self): return all(node.wait_for_verack() for node in self.test_nodes) @@ -197,7 +197,7 @@ class TestManager(object): def wait_for_pings(self, counter): def received_pongs(): return all(node.received_ping_response(counter) for node in self.test_nodes) - return wait_until(received_pongs) + wait_until(received_pongs, lock=mininode_lock) # sync_blocks: Wait for all connections to request the blockhash given # then send get_headers to find out the tip of each node, and synchronize @@ -210,8 +210,7 @@ class TestManager(object): ) # --> error if not requested - if not wait_until(blocks_requested, attempts=20*num_blocks): - raise AssertionError("Not all nodes requested block") + wait_until(blocks_requested, attempts=20*num_blocks, lock=mininode_lock) # Send getheaders message [ c.cb.send_getheaders() for c in self.connections ] @@ -231,8 +230,7 @@ class TestManager(object): ) # --> error if not requested - if not wait_until(transaction_requested, attempts=20*num_events): - raise AssertionError("Not all nodes requested transaction") + wait_until(transaction_requested, attempts=20*num_events, lock=mininode_lock) # Get the mempool [ c.cb.send_mempool() for c in self.connections ] diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index a4d85501e7..843b67cd5c 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -35,7 +35,7 @@ import time from threading import RLock, Thread from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until BIP0031_VERSION = 60000 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -1358,23 +1358,6 @@ class msg_reject(object): return "msg_reject: %s %d %s [%064x]" \ % (self.message, self.code, self.reason, self.data) -# Helper function -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')): - if attempts == float('inf') and timeout == float('inf'): - timeout = 60 - attempt = 0 - elapsed = 0 - - while attempt < attempts and elapsed < timeout: - with mininode_lock: - if predicate(): - return True - attempt += 1 - elapsed += 0.05 - time.sleep(0.05) - - return False - class msg_feefilter(object): command = b"feefilter" @@ -1522,6 +1505,7 @@ class NodeConnCB(object): except: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) + raise def set_deliver_sleep_time(self, value): with mininode_lock: @@ -1591,21 +1575,21 @@ class NodeConnCB(object): def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.connected - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message receiving helper methods def wait_for_block(self, blockhash, timeout=60): test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_getdata(self, timeout=60): test_function = lambda: self.last_message.get("getdata") - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_getheaders(self, timeout=60): test_function = lambda: self.last_message.get("getheaders") - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_inv(self, expected_inv, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" @@ -1614,11 +1598,11 @@ class NodeConnCB(object): test_function = lambda: self.last_message.get("inv") and \ self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) def wait_for_verack(self, timeout=60): test_function = lambda: self.message_count["verack"] - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message sending helper functions @@ -1636,7 +1620,7 @@ class NodeConnCB(object): def sync_with_ping(self, timeout=60): self.send_message(msg_ping(nonce=self.ping_counter)) test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter - assert wait_until(test_function, timeout=timeout) + wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 return True @@ -1725,13 +1709,10 @@ class NodeConn(asyncore.dispatcher): self.cb.on_close(self) def handle_read(self): - try: - t = self.recv(8192) - if len(t) > 0: - self.recvbuf += t - self.got_data() - except: - pass + t = self.recv(8192) + if len(t) > 0: + self.recvbuf += t + self.got_data() def readable(self): return True @@ -1797,8 +1778,10 @@ class NodeConn(asyncore.dispatcher): self.got_message(t) else: logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + raise ValueError("Unknown command: '%s'" % (command)) except Exception as e: logger.exception('got_data:', repr(e)) + raise def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: @@ -1854,6 +1837,7 @@ class NetworkThread(Thread): disconnected.append(obj) [ obj.handle_close() for obj in disconnected ] asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1) + logger.debug("Network thread closing") # An exception we can raise if we detect a potential disconnect diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index e562d11938..a53eb51799 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -5,15 +5,12 @@ """Base class for RPC testing.""" from collections import deque -import errno from enum import Enum -import http.client import logging import optparse import os import pdb import shutil -import subprocess import sys import tempfile import time @@ -21,6 +18,7 @@ import traceback from .authproxy import JSONRPCException from . import coverage +from .test_node import TestNode from .util import ( MAX_NODES, PortSeed, @@ -28,12 +26,9 @@ from .util import ( check_json_precision, connect_nodes_bi, disconnect_nodes, - get_rpc_proxy, initialize_datadir, - get_datadir_path, log_filename, p2p_port, - rpc_url, set_node_times, sync_blocks, sync_mempools, @@ -48,63 +43,33 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 -BITCOIND_PROC_WAIT_TIMEOUT = 60 - class BitcoinTestFramework(object): """Base class for a bitcoin test script. - Individual bitcoin test scripts should subclass this class and override the following methods: + Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. + + Individual tests can also override the following methods to customize the test setup: - - __init__() - add_options() - setup_chain() - setup_network() - - run_test() + - setup_nodes() - The main() method should not be overridden. + The __init__() and main() methods should not be overridden. This class also contains various public and private helper methods.""" - # Methods to override in subclass test scripts. def __init__(self): - self.num_nodes = 4 + """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.setup_clean_chain = False self.nodes = [] - self.bitcoind_processes = {} self.mocktime = 0 + self.set_test_params() - def add_options(self, parser): - pass - - def setup_chain(self): - self.log.info("Initializing test directory " + self.options.tmpdir) - if self.setup_clean_chain: - self._initialize_chain_clean(self.options.tmpdir, self.num_nodes) - else: - self._initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir) - - def setup_network(self): - self.setup_nodes() - - # Connect the nodes as a "chain". This allows us - # to split the network between nodes 1 and 2 to get - # two halves that can work on competing chains. - for i in range(self.num_nodes - 1): - connect_nodes_bi(self.nodes, i, i + 1) - self.sync_all() - - def setup_nodes(self): - extra_args = None - if hasattr(self, "extra_args"): - extra_args = self.extra_args - self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) - - def run_test(self): - raise NotImplementedError - - # Main function. This should not be overridden by the subclass test scripts. + assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" def main(self): + """Main function. This should not be overridden by the subclass test scripts.""" parser = optparse.OptionParser(usage="%prog [options]") parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", @@ -208,77 +173,115 @@ class BitcoinTestFramework(object): logging.shutdown() sys.exit(TEST_EXIT_FAILED) - # Public helper methods. These can be accessed by the subclass test scripts. + # Methods to override in subclass test scripts. + def set_test_params(self): + """Tests must this method to change default values for number of nodes, topology, etc""" + raise NotImplementedError - def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - """Start a bitcoind and return RPC connection to it""" + def add_options(self, parser): + """Override this method to add command-line options to the test""" + pass - datadir = os.path.join(dirname, "node" + str(i)) - if binary is None: - binary = os.getenv("BITCOIND", "bitcoind") - args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(self.mocktime), "-uacomment=testnode%d" % i] - if extra_args is not None: - args.extend(extra_args) - self.bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) - self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up") - self._wait_for_bitcoind_start(self.bitcoind_processes[i], datadir, i, rpchost) - self.log.debug("initialize_chain: RPC successfully started") - proxy = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, timeout=timewait) + def setup_chain(self): + """Override this method to customize blockchain setup""" + self.log.info("Initializing test directory " + self.options.tmpdir) + if self.setup_clean_chain: + self._initialize_chain_clean() + else: + self._initialize_chain() - if self.options.coveragedir: - coverage.write_all_rpc_commands(self.options.coveragedir, proxy) + def setup_network(self): + """Override this method to customize test network topology""" + self.setup_nodes() + + # Connect the nodes as a "chain". This allows us + # to split the network between nodes 1 and 2 to get + # two halves that can work on competing chains. + for i in range(self.num_nodes - 1): + connect_nodes_bi(self.nodes, i, i + 1) + self.sync_all() + + def setup_nodes(self): + """Override this method to customize test node setup""" + extra_args = None + if hasattr(self, "extra_args"): + extra_args = self.extra_args + self.add_nodes(self.num_nodes, extra_args) + self.start_nodes() + + def run_test(self): + """Tests must override this method to define test logic""" + raise NotImplementedError - return proxy + # Public helper methods. These can be accessed by the subclass test scripts. - def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """Start multiple bitcoinds, return RPC connections to them""" + def add_nodes(self, num_nodes, extra_args=None, rpchost=None, timewait=None, binary=None): + """Instantiate TestNode objects""" if extra_args is None: - extra_args = [None] * num_nodes + extra_args = [[]] * num_nodes if binary is None: binary = [None] * num_nodes assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) - rpcs = [] + for i in range(num_nodes): + self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir)) + + def start_node(self, i, extra_args=None, stderr=None): + """Start a bitcoind""" + + node = self.nodes[i] + + node.start(extra_args, stderr) + node.wait_for_rpc_connection() + + if self.options.coveragedir is not None: + coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) + + def start_nodes(self, extra_args=None): + """Start multiple bitcoinds""" + + if extra_args is None: + extra_args = [None] * self.num_nodes + assert_equal(len(extra_args), self.num_nodes) try: - for i in range(num_nodes): - rpcs.append(self.start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) + for i, node in enumerate(self.nodes): + node.start(extra_args[i]) + for node in self.nodes: + node.wait_for_rpc_connection() except: # If one node failed to start, stop the others - # TODO: abusing self.nodes in this way is a little hacky. - # Eventually we should do a better job of tracking nodes - self.nodes.extend(rpcs) self.stop_nodes() - self.nodes = [] raise - return rpcs + + if self.options.coveragedir is not None: + for node in self.nodes: + coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) def stop_node(self, i): """Stop a bitcoind test node""" - - self.log.debug("Stopping node %d" % i) - try: - self.nodes[i].stop() - except http.client.CannotSendRequest as e: - self.log.exception("Unable to stop node") - return_code = self.bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - del self.bitcoind_processes[i] - assert_equal(return_code, 0) + self.nodes[i].stop_node() + self.nodes[i].wait_until_stopped() def stop_nodes(self): """Stop multiple bitcoind test nodes""" + for node in self.nodes: + # Issue RPC to stop nodes + node.stop_node() - for i in range(len(self.nodes)): - self.stop_node(i) - assert not self.bitcoind_processes.values() # All connections must be gone now + for node in self.nodes: + # Wait for nodes to stop + node.wait_until_stopped() - def assert_start_raises_init_error(self, i, dirname, extra_args=None, expected_msg=None): + def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: - self.start_node(i, dirname, extra_args, stderr=log_stderr) + self.start_node(i, extra_args, stderr=log_stderr) self.stop_node(i) except Exception as e: assert 'bitcoind exited' in str(e) # node must have shutdown + self.nodes[i].running = False + self.nodes[i].process = None if expected_msg is not None: log_stderr.seek(0) stderr = log_stderr.read().decode('utf-8') @@ -292,7 +295,7 @@ class BitcoinTestFramework(object): raise AssertionError(assert_msg) def wait_for_node_exit(self, i, timeout): - self.bitcoind_processes[i].wait(timeout) + self.nodes[i].process.wait(timeout) def split_network(self): """ @@ -362,16 +365,16 @@ class BitcoinTestFramework(object): rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) - def _initialize_chain(self, test_dir, num_nodes, cachedir): + def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. Create a cache of a 200-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" - assert num_nodes <= MAX_NODES + assert self.num_nodes <= MAX_NODES create_cache = False for i in range(MAX_NODES): - if not os.path.isdir(os.path.join(cachedir, 'node' + str(i))): + if not os.path.isdir(os.path.join(self.options.cachedir, 'node' + str(i))): create_cache = True break @@ -380,27 +383,22 @@ class BitcoinTestFramework(object): # find and delete old cache directories if any exist for i in range(MAX_NODES): - if os.path.isdir(os.path.join(cachedir, "node" + str(i))): - shutil.rmtree(os.path.join(cachedir, "node" + str(i))) + if os.path.isdir(os.path.join(self.options.cachedir, "node" + str(i))): + shutil.rmtree(os.path.join(self.options.cachedir, "node" + str(i))) # Create cache directories, run bitcoinds: for i in range(MAX_NODES): - datadir = initialize_datadir(cachedir, i) + datadir = initialize_datadir(self.options.cachedir, i) args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) - self.bitcoind_processes[i] = subprocess.Popen(args) - self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up") - self._wait_for_bitcoind_start(self.bitcoind_processes[i], datadir, i) - self.log.debug("initialize_chain: RPC successfully started") + self.nodes.append(TestNode(i, self.options.cachedir, extra_args=[], rpchost=None, timewait=None, binary=None, stderr=None, mocktime=self.mocktime, coverage_dir=None)) + self.nodes[i].args = args + self.start_node(i) - self.nodes = [] - for i in range(MAX_NODES): - try: - self.nodes.append(get_rpc_proxy(rpc_url(get_datadir_path(cachedir, i), i), i)) - except: - self.log.exception("Error connecting to node %d" % i) - sys.exit(1) + # Wait for RPC connections to be ready + for node in self.nodes: + node.wait_for_rpc_connection() # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. @@ -425,48 +423,24 @@ class BitcoinTestFramework(object): self.nodes = [] self.disable_mocktime() for i in range(MAX_NODES): - os.remove(log_filename(cachedir, i, "debug.log")) - os.remove(log_filename(cachedir, i, "db.log")) - os.remove(log_filename(cachedir, i, "peers.dat")) - os.remove(log_filename(cachedir, i, "fee_estimates.dat")) - - for i in range(num_nodes): - from_dir = os.path.join(cachedir, "node" + str(i)) - to_dir = os.path.join(test_dir, "node" + str(i)) + os.remove(log_filename(self.options.cachedir, i, "debug.log")) + os.remove(log_filename(self.options.cachedir, i, "db.log")) + os.remove(log_filename(self.options.cachedir, i, "peers.dat")) + os.remove(log_filename(self.options.cachedir, i, "fee_estimates.dat")) + + for i in range(self.num_nodes): + from_dir = os.path.join(self.options.cachedir, "node" + str(i)) + to_dir = os.path.join(self.options.tmpdir, "node" + str(i)) shutil.copytree(from_dir, to_dir) - initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf + initialize_datadir(self.options.tmpdir, i) # Overwrite port/rpcport in bitcoin.conf - def _initialize_chain_clean(self, test_dir, num_nodes): + def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" - for i in range(num_nodes): - initialize_datadir(test_dir, i) - - def _wait_for_bitcoind_start(self, process, datadir, i, rpchost=None): - """Wait for bitcoind to start. - - This means that RPC is accessible and fully initialized. - Raise an exception if bitcoind exits during initialization.""" - while True: - if process.poll() is not None: - raise Exception('bitcoind exited with status %i during initialization' % process.returncode) - try: - # Check if .cookie file to be created - rpc = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, coveragedir=self.options.coveragedir) - rpc.getblockcount() - break # break out of loop on success - except IOError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? - raise # unknown IO error - except JSONRPCException as e: # Initialization phase - if e.error['code'] != -28: # RPC in warmup? - raise # unknown JSON RPC exception - except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting - if "No RPC credentials" not in str(e): - raise - time.sleep(0.25) + for i in range(self.num_nodes): + initialize_datadir(self.options.tmpdir, i) class ComparisonTestFramework(BitcoinTestFramework): """Test framework for doing p2p comparison testing @@ -476,8 +450,7 @@ class ComparisonTestFramework(BitcoinTestFramework): - 2 binaries: 1 test binary, 1 ref binary - n>2 binaries: 1 test binary, n-1 ref binaries""" - def __init__(self): - super().__init__() + def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True @@ -490,13 +463,13 @@ class ComparisonTestFramework(BitcoinTestFramework): help="bitcoind binary to use for reference nodes (if any)") def setup_network(self): - extra_args = [['-whitelist=127.0.0.1']]*self.num_nodes + extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes if hasattr(self, "extra_args"): extra_args = self.extra_args - self.nodes = self.start_nodes( - self.num_nodes, self.options.tmpdir, extra_args, - binary=[self.options.testbinary] + - [self.options.refbinary] * (self.num_nodes - 1)) + self.add_nodes(self.num_nodes, extra_args, + binary=[self.options.testbinary] + + [self.options.refbinary] * (self.num_nodes - 1)) + self.start_nodes() class SkipTest(Exception): """This exception is raised to skip a test""" diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py new file mode 100755 index 0000000000..12dab57a02 --- /dev/null +++ b/test/functional/test_framework/test_node.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Class for bitcoind node under test""" + +import decimal +import errno +import http.client +import json +import logging +import os +import subprocess +import time + +from .util import ( + assert_equal, + get_rpc_proxy, + rpc_url, + wait_until, +) +from .authproxy import JSONRPCException + +BITCOIND_PROC_WAIT_TIMEOUT = 60 + +class TestNode(): + """A class for representing a bitcoind node under test. + + This class contains: + + - state about the node (whether it's running, etc) + - a Python subprocess.Popen object representing the running process + - an RPC connection to the node + + To make things easier for the test writer, a bit of magic is happening under the covers. + Any unrecognised messages will be dispatched to the RPC connection.""" + + def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): + self.index = i + self.datadir = os.path.join(dirname, "node" + str(i)) + self.rpchost = rpchost + if timewait: + self.rpc_timeout = timewait + else: + # Wait for up to 60 seconds for the RPC server to respond + self.rpc_timeout = 60 + if binary is None: + self.binary = os.getenv("BITCOIND", "bitcoind") + else: + self.binary = binary + self.stderr = stderr + self.coverage_dir = coverage_dir + # Most callers will just need to add extra args to the standard list below. For those callers that need more flexibity, they can just set the args property directly. + self.extra_args = extra_args + self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i] + + self.cli = TestNodeCLI(os.getenv("BITCOINCLI", "bitcoin-cli"), self.datadir) + + self.running = False + self.process = None + self.rpc_connected = False + self.rpc = None + self.url = None + self.log = logging.getLogger('TestFramework.node%d' % i) + + def __getattr__(self, *args, **kwargs): + """Dispatches any unrecognised messages to the RPC connection.""" + assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" + return self.rpc.__getattr__(*args, **kwargs) + + def start(self, extra_args=None, stderr=None): + """Start the node.""" + if extra_args is None: + extra_args = self.extra_args + if stderr is None: + stderr = self.stderr + self.process = subprocess.Popen(self.args + extra_args, stderr=stderr) + self.running = True + self.log.debug("bitcoind started, waiting for RPC to come up") + + def wait_for_rpc_connection(self): + """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" + # Poll at a rate of four times per second + poll_per_s = 4 + for _ in range(poll_per_s * self.rpc_timeout): + assert self.process.poll() is None, "bitcoind exited with status %i during initialization" % self.process.returncode + try: + self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + self.rpc.getblockcount() + # If the call to getblockcount() succeeds then the RPC connection is up + self.rpc_connected = True + self.url = self.rpc.url + self.log.debug("RPC successfully started") + return + except IOError as e: + if e.errno != errno.ECONNREFUSED: # Port not yet open? + raise # unknown IO error + except JSONRPCException as e: # Initialization phase + if e.error['code'] != -28: # RPC in warmup? + raise # unknown JSON RPC exception + except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting + if "No RPC credentials" not in str(e): + raise + time.sleep(1.0 / poll_per_s) + raise AssertionError("Unable to connect to bitcoind") + + def get_wallet_rpc(self, wallet_name): + assert self.rpc_connected + assert self.rpc + wallet_path = "wallet/%s" % wallet_name + return self.rpc / wallet_path + + def stop_node(self): + """Stop the node.""" + if not self.running: + return + self.log.debug("Stopping node") + try: + self.stop() + except http.client.CannotSendRequest: + self.log.exception("Unable to stop node.") + + def is_node_stopped(self): + """Checks whether the node has stopped. + + Returns True if the node has stopped. False otherwise. + This method is responsible for freeing resources (self.process).""" + if not self.running: + return True + return_code = self.process.poll() + if return_code is None: + return False + + # process has stopped. Assert that it didn't return an error code. + assert_equal(return_code, 0) + self.running = False + self.process = None + self.rpc_connected = False + self.rpc = None + self.log.debug("Node stopped") + return True + + def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): + wait_until(self.is_node_stopped, timeout=timeout) + + def node_encrypt_wallet(self, passphrase): + """"Encrypts the wallet. + + This causes bitcoind to shutdown, so this method takes + care of cleaning up resources.""" + self.encryptwallet(passphrase) + self.wait_until_stopped() + +class TestNodeCLI(): + """Interface to bitcoin-cli for an individual node""" + + def __init__(self, binary, datadir): + self.args = [] + self.binary = binary + self.datadir = datadir + self.input = None + + def __call__(self, *args, input=None): + # TestNodeCLI is callable with bitcoin-cli command-line args + self.args = [str(arg) for arg in args] + self.input = input + return self + + def __getattr__(self, command): + def dispatcher(*args, **kwargs): + return self.send_cli(command, *args, **kwargs) + return dispatcher + + def send_cli(self, command, *args, **kwargs): + """Run bitcoin-cli command. Deserializes returned string as python object.""" + + pos_args = [str(arg) for arg in args] + named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()] + assert not (pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call" + p_args = [self.binary, "-datadir=" + self.datadir] + self.args + if named_args: + p_args += ["-named"] + p_args += [command] + pos_args + named_args + process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + cli_stdout, cli_stderr = process.communicate(input=self.input) + returncode = process.poll() + if returncode: + # Ignore cli_stdout, raise with cli_stderr + raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) + return json.loads(cli_stdout, parse_float=decimal.Decimal) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index acca72aa86..64966adb97 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -7,11 +7,13 @@ from base64 import b64encode from binascii import hexlify, unhexlify from decimal import Decimal, ROUND_DOWN +import hashlib import json import logging import os import random import re +from subprocess import CalledProcessError import time from . import coverage @@ -57,18 +59,42 @@ def assert_raises_message(exc, message, fun, *args, **kwds): else: raise AssertionError("No exception raised") +def assert_raises_process_error(returncode, output, fun, *args, **kwds): + """Execute a process and asserts the process return code and output. + + Calls function `fun` with arguments `args` and `kwds`. Catches a CalledProcessError + and verifies that the return code and output are as expected. Throws AssertionError if + no CalledProcessError was raised or if the return code and output are not as expected. + + Args: + returncode (int): the process return code. + output (string): [a substring of] the process output. + fun (function): the function to call. This should execute a process. + args*: positional arguments for the function. + kwds**: named arguments for the function. + """ + try: + fun(*args, **kwds) + except CalledProcessError as e: + if returncode != e.returncode: + raise AssertionError("Unexpected returncode %i" % e.returncode) + if output not in e.output: + raise AssertionError("Expected substring not found:" + e.output) + else: + raise AssertionError("No exception raised") + def assert_raises_jsonrpc(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException and verifies that the error code and message are as expected. Throws AssertionError if - no JSONRPCException was returned or if the error code/message are not as expected. + no JSONRPCException was raised or if the error code/message are not as expected. Args: code (int), optional: the error code returned by the RPC call (defined in src/rpc/protocol.h). Set to None if checking the error code is not required. message (string), optional: [a substring of] the error string returned by the - RPC call. Set to None if checking the error string is not required + RPC call. Set to None if checking the error string is not required. fun (function): the function to call. This should be the name of an RPC. args*: positional arguments for the function. kwds**: named arguments for the function. @@ -148,6 +174,13 @@ def count_bytes(hex_string): def bytes_to_hex_str(byte_str): return hexlify(byte_str).decode('ascii') +def hash256(byte_str): + sha256 = hashlib.sha256() + sha256.update(byte_str) + sha256d = hashlib.sha256() + sha256d.update(sha256.digest()) + return sha256d.digest()[::-1] + def hex_str_to_bytes(hex_str): return unhexlify(hex_str.encode('ascii')) @@ -157,6 +190,28 @@ def str_to_b64str(string): def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None): + if attempts == float('inf') and timeout == float('inf'): + timeout = 60 + attempt = 0 + timeout += time.time() + + while attempt < attempts and time.time() < timeout: + if lock: + with lock: + if predicate(): + return + else: + if predicate(): + return + attempt += 1 + time.sleep(0.05) + + # Print the cause of the timeout + assert_greater_than(attempts, attempt) + assert_greater_than(timeout, time.time()) + raise RuntimeError('Unreachable') + # RPC/P2P connection constants and functions ############################################ @@ -204,7 +259,7 @@ def rpc_port(n): return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_url(datadir, i, rpchost=None): - rpc_u, rpc_p = get_auth_cookie(datadir, i) + rpc_u, rpc_p = get_auth_cookie(datadir) host = '127.0.0.1' port = rpc_port(i) if rpchost: @@ -232,7 +287,7 @@ def initialize_datadir(dirname, n): def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) -def get_auth_cookie(datadir, n): +def get_auth_cookie(datadir): user = None password = None if os.path.isfile(os.path.join(datadir, "bitcoin.conf")): |