diff options
Diffstat (limited to 'qa/rpc-tests/test_framework')
-rw-r--r-- | qa/rpc-tests/test_framework/address.py | 7 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/authproxy.py | 67 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/bignum.py | 10 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/blockstore.py | 18 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/blocktools.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/comptool.py | 50 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/coverage.py | 9 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/key.py | 10 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/mininode.py | 96 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/netutil.py | 5 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/script.py | 12 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/siphash.py | 7 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/socks5.py | 14 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/test_framework.py | 71 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/util.py | 154 |
15 files changed, 248 insertions, 284 deletions
diff --git a/qa/rpc-tests/test_framework/address.py b/qa/rpc-tests/test_framework/address.py index 50b999be61..96bebe1ea1 100644 --- a/qa/rpc-tests/test_framework/address.py +++ b/qa/rpc-tests/test_framework/address.py @@ -2,12 +2,7 @@ # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# -# address.py -# -# This file encodes and decodes BASE58 P2PKH and P2SH addresses -# +"""Encode and decode BASE58, P2PKH and P2SH addresses.""" from .script import hash256, hash160, sha256, CScript, OP_0 from .util import bytes_to_hex_str, hex_str_to_bytes diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 09ed611299..9ab3094b06 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -1,37 +1,36 @@ - -""" - Copyright (c) 2011 Jeff Garzik - - AuthServiceProxy has the following improvements over python-jsonrpc's - ServiceProxy class: - - - HTTP connections persist for the life of the AuthServiceProxy object - (if server supports HTTP/1.1) - - sends protocol 'version', per JSON-RPC 1.1 - - sends proper, incrementing 'id' - - sends Basic HTTP authentication headers - - parses all JSON numbers that look like floats as Decimal - - uses standard Python json lib - - Previous copyright, from python-jsonrpc/jsonrpc/proxy.py: - - Copyright (c) 2007 Jan-Klaas Kollhof - - This file is part of jsonrpc. - - jsonrpc is free software; you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 2.1 of the License, or - (at your option) any later version. - - This software is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this software; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Copyright (c) 2011 Jeff Garzik +# +# Previous copyright, from python-jsonrpc/jsonrpc/proxy.py: +# +# Copyright (c) 2007 Jan-Klaas Kollhof +# +# This file is part of jsonrpc. +# +# jsonrpc is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this software; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +"""HTTP proxy for opening RPC connection to bitcoind. + +AuthServiceProxy has the following improvements over python-jsonrpc's +ServiceProxy class: + +- HTTP connections persist for the life of the AuthServiceProxy object + (if server supports HTTP/1.1) +- sends protocol 'version', per JSON-RPC 1.1 +- sends proper, incrementing 'id' +- sends Basic HTTP authentication headers +- parses all JSON numbers that look like floats as Decimal +- uses standard Python json lib """ try: diff --git a/qa/rpc-tests/test_framework/bignum.py b/qa/rpc-tests/test_framework/bignum.py index ef800e4d57..024611da6e 100644 --- a/qa/rpc-tests/test_framework/bignum.py +++ b/qa/rpc-tests/test_framework/bignum.py @@ -1,15 +1,11 @@ #!/usr/bin/env python3 # -# bignum.py -# -# This file is copied from python-bitcoinlib. -# # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -"""Bignum routines""" +"""Big number routines. +This file is copied from python-bitcoinlib. +""" import struct diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py index 28a6b92b81..4cfd682bb5 100644 --- a/qa/rpc-tests/test_framework/blockstore.py +++ b/qa/rpc-tests/test_framework/blockstore.py @@ -2,16 +2,22 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# BlockStore: a helper class that keeps a map of blocks and implements -# helper functions for responding to getheaders and getdata, -# and for constructing a getheaders message -# +"""BlockStore and TxStore helper classes.""" from .mininode import * from io import BytesIO import dbm.dumb as dbmd +logger = logging.getLogger("TestFramework.blockstore") + class BlockStore(object): + """BlockStore helper class. + + BlockStore keeps a map of blocks and implements helper functions for + responding to getheaders and getdata, and for constructing a getheaders + message. + """ + def __init__(self, datadir): self.blockDB = dbmd.open(datadir + "/blocks", 'c') self.currentBlock = 0 @@ -82,7 +88,7 @@ class BlockStore(object): try: self.blockDB[repr(block.sha256)] = bytes(block.serialize()) except TypeError as e: - print("Unexpected error: ", sys.exc_info()[0], e.args) + logger.exception("Unexpected error") self.currentBlock = block.sha256 self.headers_map[block.sha256] = CBlockHeader(block) @@ -152,7 +158,7 @@ class TxStore(object): try: self.txDB[repr(tx.sha256)] = bytes(tx.serialize()) except TypeError as e: - print("Unexpected error: ", sys.exc_info()[0], e.args) + logger.exception("Unexpected error") def get_transactions(self, inv): responses = [] diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py index f69958823c..2c9a0857df 100644 --- a/qa/rpc-tests/test_framework/blocktools.py +++ b/qa/rpc-tests/test_framework/blocktools.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -# blocktools.py - utilities for manipulating blocks and transactions # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Utilities for manipulating blocks and transactions.""" from .mininode import * from .script import CScript, OP_TRUE, OP_CHECKSIG, OP_RETURN diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py index 17679fc7e1..70d1d700ef 100755 --- a/qa/rpc-tests/test_framework/comptool.py +++ b/qa/rpc-tests/test_framework/comptool.py @@ -2,34 +2,33 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Compare two or more bitcoinds to each other. + +To use, create a class that implements get_tests(), and pass it in +as the test generator to TestManager. get_tests() should be a python +generator that returns TestInstance objects. See below for definition. + +TestNode behaves as follows: + Configure with a BlockStore and TxStore + on_inv: log the message but don't request + on_headers: log the chain tip + on_pong: update ping response map (for synchronization) + on_getheaders: provide headers via BlockStore + on_getdata: provide blocks via BlockStore +""" from .mininode import * from .blockstore import BlockStore, TxStore from .util import p2p_port -''' -This is a tool for comparing two or more bitcoinds to each other -using a script provided. - -To use, create a class that implements get_tests(), and pass it in -as the test generator to TestManager. get_tests() should be a python -generator that returns TestInstance objects. See below for definition. -''' +import logging -# TestNode behaves as follows: -# Configure with a BlockStore and TxStore -# on_inv: log the message but don't request -# on_headers: log the chain tip -# on_pong: update ping response map (for synchronization) -# on_getheaders: provide headers via BlockStore -# on_getdata: provide blocks via BlockStore +logger=logging.getLogger("TestFramework.comptool") global mininode_lock class RejectResult(object): - ''' - Outcome that expects rejection of a transaction or block. - ''' + """Outcome that expects rejection of a transaction or block.""" def __init__(self, code, reason=b''): self.code = code self.reason = reason @@ -214,7 +213,6 @@ class TestManager(object): # --> error if not requested if not wait_until(blocks_requested, attempts=20*num_blocks): - # print [ c.cb.block_request_map for c in self.connections ] raise AssertionError("Not all nodes requested block") # Send getheaders message @@ -236,7 +234,6 @@ class TestManager(object): # --> error if not requested if not wait_until(transaction_requested, attempts=20*num_events): - # print [ c.cb.tx_request_map for c in self.connections ] raise AssertionError("Not all nodes requested transaction") # Get the mempool @@ -263,13 +260,12 @@ class TestManager(object): if c.cb.bestblockhash == blockhash: return False if blockhash not in c.cb.block_reject_map: - print('Block not in reject map: %064x' % (blockhash)) + logger.error('Block not in reject map: %064x' % (blockhash)) return False if not outcome.match(c.cb.block_reject_map[blockhash]): - print('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) + logger.error('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) return False elif ((c.cb.bestblockhash == blockhash) != outcome): - # print c.cb.bestblockhash, blockhash, outcome return False return True @@ -285,19 +281,17 @@ class TestManager(object): if outcome is None: # Make sure the mempools agree with each other if c.cb.lastInv != self.connections[0].cb.lastInv: - # print c.rpc.getrawmempool() return False elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code if txhash in c.cb.lastInv: return False if txhash not in c.cb.tx_reject_map: - print('Tx not in reject map: %064x' % (txhash)) + logger.error('Tx not in reject map: %064x' % (txhash)) return False if not outcome.match(c.cb.tx_reject_map[txhash]): - print('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) + logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) return False elif ((txhash in c.cb.lastInv) != outcome): - # print c.rpc.getrawmempool(), c.cb.lastInv return False return True @@ -407,7 +401,7 @@ class TestManager(object): if (not self.check_mempool(tx.sha256, tx_outcome)): raise AssertionError("Mempool test failed at test %d" % test_number) - print("Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ]) + logger.info("Test %d: PASS" % test_number) test_number += 1 [ c.disconnect_node() for c in self.connections ] diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py index 13b33869f5..3f87ef91f6 100644 --- a/qa/rpc-tests/test_framework/coverage.py +++ b/qa/rpc-tests/test_framework/coverage.py @@ -2,15 +2,12 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Utilities for doing coverage analysis on the RPC interface. -""" -This module contains utilities for doing coverage analysis on the RPC -interface. - -It provides a way to track which RPC commands are exercised during +Provides a way to track which RPC commands are exercised during testing. - """ + import os diff --git a/qa/rpc-tests/test_framework/key.py b/qa/rpc-tests/test_framework/key.py index c63a15c1e0..85a6158a2f 100644 --- a/qa/rpc-tests/test_framework/key.py +++ b/qa/rpc-tests/test_framework/key.py @@ -1,14 +1,10 @@ # Copyright (c) 2011 Sam Rushing -# -# key.py - OpenSSL wrapper -# -# This file is modified from python-bitcoinlib. -# - -"""ECC secp256k1 crypto routines +"""ECC secp256k1 OpenSSL wrapper. WARNING: This module does not mlock() secrets; your private keys may end up on disk in swap! Use with caution! + +This file is modified from python-bitcoinlib. """ import ctypes diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 91daa4ab1f..aace17a043 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -4,23 +4,21 @@ # Copyright (c) 2010-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# -# mininode.py - Bitcoin P2P network half-a-node -# -# This python code was modified from ArtForz' public domain half-a-node, as -# found in the mini-node branch of http://github.com/jgarzik/pynode. -# -# NodeConn: an object which manages p2p connectivity to a bitcoin node -# NodeConnCB: a base class that describes the interface for receiving -# callbacks with network messages from a NodeConn -# CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: -# data structures that should map to corresponding structures in -# bitcoin/primitives -# msg_block, msg_tx, msg_headers, etc.: -# data structures that represent network messages -# ser_*, deser_*: functions that handle serialization/deserialization - +"""Bitcoin P2P network half-a-node. + +This python code was modified from ArtForz' public domain half-a-node, as +found in the mini-node branch of http://github.com/jgarzik/pynode. + +NodeConn: an object which manages p2p connectivity to a bitcoin node +NodeConnCB: a base class that describes the interface for receiving + callbacks with network messages from a NodeConn +CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: + data structures that should map to corresponding structures in + bitcoin/primitives +msg_block, msg_tx, msg_headers, etc.: + data structures that represent network messages +ser_*, deser_*: functions that handle serialization/deserialization +""" import struct import socket @@ -53,6 +51,8 @@ NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) +logger = logging.getLogger("TestFramework.mininode") + # Keep our own socket map for asyncore, so that we can track disconnects # ourselves (to workaround an issue with closing an asyncore socket when # using select) @@ -1504,8 +1504,7 @@ class NodeConnCB(object): try: getattr(self, 'on_' + message.command.decode('ascii'))(conn, message) except: - print("ERROR delivering %s (%s)" % (repr(message), - sys.exc_info()[0])) + logger.exception("ERROR delivering %s" % repr(message)) def on_version(self, conn, message): if message.nVersion >= 209: @@ -1540,6 +1539,7 @@ class NodeConnCB(object): if conn.ver_send > BIP0031_VERSION: conn.send_message(msg_pong(message.nonce)) def on_reject(self, conn, message): pass + def on_open(self, conn): pass def on_close(self, conn): pass def on_mempool(self, conn): pass def on_pong(self, conn, message): pass @@ -1614,9 +1614,8 @@ class NodeConn(asyncore.dispatcher): "regtest": b"\xfa\xbf\xb5\xda", # regtest } - def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK): + def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True): asyncore.dispatcher.__init__(self, map=mininode_socket_map) - self.log = logging.getLogger("NodeConn(%s:%d)" % (dstaddr, dstport)) self.dstaddr = dstaddr self.dstport = dstport self.create_socket(socket.AF_INET, socket.SOCK_STREAM) @@ -1631,16 +1630,17 @@ class NodeConn(asyncore.dispatcher): self.disconnect = False self.nServices = 0 - # stuff version msg into sendbuf - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.send_message(vt, True) - print('MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \ - + str(dstport)) + if send_version: + # stuff version msg into sendbuf + vt = msg_version() + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.send_message(vt, True) + + logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) try: self.connect((dstaddr, dstport)) @@ -1648,16 +1648,14 @@ class NodeConn(asyncore.dispatcher): self.handle_close() self.rpc = rpc - def show_debug_msg(self, msg): - self.log.debug(msg) - def handle_connect(self): - self.show_debug_msg("MiniNode: Connected & Listening: \n") - self.state = "connected" + if self.state != "connected": + logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) + self.state = "connected" + self.cb.on_open(self) def handle_close(self): - self.show_debug_msg("MiniNode: Closing Connection to %s:%d... " - % (self.dstaddr, self.dstport)) + logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) self.state = "closed" self.recvbuf = b"" self.sendbuf = b"" @@ -1681,11 +1679,20 @@ class NodeConn(asyncore.dispatcher): def writable(self): with mininode_lock: + pre_connection = self.state == "connecting" length = len(self.sendbuf) - return (length > 0) + return (length > 0 or pre_connection) def handle_write(self): with mininode_lock: + # asyncore does not expose socket connection, only the first read/write + # event, thus we must check connection manually here to know when we + # actually connect + if self.state == "connecting": + self.handle_connect() + if not self.writable(): + return + try: sent = self.send(self.sendbuf) except: @@ -1730,17 +1737,14 @@ class NodeConn(asyncore.dispatcher): t.deserialize(f) self.got_message(t) else: - self.show_debug_msg("Unknown command: '" + command + "' " + - repr(msg)) + logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) except Exception as e: - print('got_data:', repr(e)) - # import traceback - # traceback.print_tb(sys.exc_info()[2]) + logger.exception('got_data:', repr(e)) def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: raise IOError('Not connected, no pushbuf') - self.show_debug_msg("Send %s" % repr(message)) + logger.debug("Send message to %s:%d: %s" % (self.dstaddr, self.dstport, repr(message))) command = message.command data = message.serialize() tmsg = self.MAGIC_BYTES[self.network] @@ -1762,7 +1766,7 @@ class NodeConn(asyncore.dispatcher): self.messagemap[b'ping'] = msg_ping_prebip31 if self.last_sent + 30 * 60 < time.time(): self.send_message(self.messagemap[b'ping']()) - self.show_debug_msg("Recv %s" % repr(message)) + logger.debug("Received message from %s:%d: %s" % (self.dstaddr, self.dstport, repr(message))) self.cb.deliver(self, message) def disconnect_node(self): diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index b92a9f6e1c..45d8e22d22 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -2,8 +2,10 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Linux network utilities. -# Linux network utilities +Roughly based on http://voorloopnul.com/blog/a-python-netstat-in-less-than-100-lines-of-code/ by Ricardo Pascal +""" import sys import socket @@ -13,7 +15,6 @@ import array import os from binascii import unhexlify, hexlify -# Roughly based on http://voorloopnul.com/blog/a-python-netstat-in-less-than-100-lines-of-code/ by Ricardo Pascal STATE_ESTABLISHED = '01' STATE_SYN_SENT = '02' STATE_SYN_RECV = '03' diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py index 83bbf20479..3d9572788e 100644 --- a/qa/rpc-tests/test_framework/script.py +++ b/qa/rpc-tests/test_framework/script.py @@ -2,19 +2,11 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Functionality to build scripts, as well as SignatureHash(). -# -# script.py -# -# This file is modified from python-bitcoinlib. -# - -"""Scripts - -Functionality to build scripts, as well as SignatureHash(). +This file is modified from python-bitcoinlib. """ - from .mininode import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string from binascii import hexlify import hashlib diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py index 9c0574bd93..f68ecad36b 100644 --- a/qa/rpc-tests/test_framework/siphash.py +++ b/qa/rpc-tests/test_framework/siphash.py @@ -2,11 +2,10 @@ # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Specialized SipHash-2-4 implementations. -# -# siphash.py - Specialized SipHash-2-4 implementations -# -# This implements SipHash-2-4 for 256-bit integers. +This implements SipHash-2-4 for 256-bit integers. +""" def rotl64(n, b): return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b diff --git a/qa/rpc-tests/test_framework/socks5.py b/qa/rpc-tests/test_framework/socks5.py index 372f5ed605..450bf3775e 100644 --- a/qa/rpc-tests/test_framework/socks5.py +++ b/qa/rpc-tests/test_framework/socks5.py @@ -2,9 +2,7 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Dummy Socks5 server for testing. -''' +"""Dummy Socks5 server for testing.""" import socket, threading, queue import traceback, sys @@ -20,7 +18,7 @@ class AddressType: ### Utility functions def recvall(s, n): - '''Receive n bytes from a socket, or fail''' + """Receive n bytes from a socket, or fail.""" rv = bytearray() while n > 0: d = s.recv(n) @@ -32,7 +30,7 @@ def recvall(s, n): ### Implementation classes class Socks5Configuration(object): - '''Proxy configuration''' + """Proxy configuration.""" def __init__(self): self.addr = None # Bind address (must be set) self.af = socket.AF_INET # Bind address family @@ -40,7 +38,7 @@ class Socks5Configuration(object): self.auth = False # Support authentication class Socks5Command(object): - '''Information about an incoming socks5 command''' + """Information about an incoming socks5 command.""" def __init__(self, cmd, atyp, addr, port, username, password): self.cmd = cmd # Command (one of Command.*) self.atyp = atyp # Address type (one of AddressType.*) @@ -58,9 +56,7 @@ class Socks5Connection(object): self.peer = peer def handle(self): - ''' - Handle socks5 request according to RFC1928 - ''' + """Handle socks5 request according to RFC192.""" try: # Verify socks version ver = recvall(self.conn, 1)[0] diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 98c4f6070b..d7072fa78d 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -2,8 +2,7 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Base class for RPC testing +"""Base class for RPC testing.""" import logging import optparse @@ -43,7 +42,7 @@ class BitcoinTestFramework(object): pass def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) + self.log.info("Initializing test directory "+self.options.tmpdir) if self.setup_clean_chain: initialize_chain_clean(self.options.tmpdir, self.num_nodes) else: @@ -113,6 +112,8 @@ class BitcoinTestFramework(object): help="Directory for caching pregenerated datadirs") parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), help="Root directory for datadirs") + parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO", + help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int', @@ -125,9 +126,6 @@ class BitcoinTestFramework(object): # backup dir variable for removal at cleanup self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed) - if self.options.trace_rpc: - logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) - if self.options.coveragedir: enable_coverage(self.options.coveragedir) @@ -137,41 +135,41 @@ class BitcoinTestFramework(object): check_json_precision() + # Set up temp directory and start logging + os.makedirs(self.options.tmpdir, exist_ok=False) + self._start_logging() + success = False + try: - os.makedirs(self.options.tmpdir, exist_ok=False) self.setup_chain() self.setup_network() self.run_test() success = True except JSONRPCException as e: - print("JSONRPC error: "+e.error['message']) - traceback.print_tb(sys.exc_info()[2]) + self.log.exception("JSONRPC error") except AssertionError as e: - print("Assertion failed: " + str(e)) - traceback.print_tb(sys.exc_info()[2]) + self.log.exception("Assertion failed") except KeyError as e: - print("key not found: "+ str(e)) - traceback.print_tb(sys.exc_info()[2]) + self.log.exception("Key error") except Exception as e: - print("Unexpected exception caught during testing: " + repr(e)) - traceback.print_tb(sys.exc_info()[2]) + self.log.exception("Unexpected exception caught during testing") except KeyboardInterrupt as e: - print("Exiting after " + repr(e)) + self.log.warning("Exiting after keyboard interrupt") if not self.options.noshutdown: - print("Stopping nodes") + self.log.info("Stopping nodes") stop_nodes(self.nodes) else: - print("Note: bitcoinds were not stopped and may still be running") + self.log.info("Note: bitcoinds were not stopped and may still be running") if not self.options.nocleanup and not self.options.noshutdown and success: - print("Cleaning up") + self.log.info("Cleaning up") shutil.rmtree(self.options.tmpdir) if not os.listdir(self.options.root): os.rmdir(self.options.root) else: - print("Not cleaning up dir %s" % self.options.tmpdir) + self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) if os.getenv("PYTHON_DEBUG", ""): # Dump the end of the debug logs, to aid in debugging rare # travis failures. @@ -183,12 +181,39 @@ class BitcoinTestFramework(object): from collections import deque print("".join(deque(open(f), MAX_LINES_TO_PRINT))) if success: - print("Tests successful") + self.log.info("Tests successful") sys.exit(0) else: - print("Failed") + self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) + logging.shutdown() sys.exit(1) + def _start_logging(self): + # Add logger and logging handlers + self.log = logging.getLogger('TestFramework') + self.log.setLevel(logging.DEBUG) + # Create file handler to log all messages + fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') + fh.setLevel(logging.DEBUG) + # Create console handler to log messages to stderr. By default this logs only error messages, but can be configured with --loglevel. + ch = logging.StreamHandler(sys.stdout) + # User can provide log level as a number or string (eg DEBUG). loglevel was caught as a string, so try to convert it to an int + ll = int(self.options.loglevel) if self.options.loglevel.isdigit() else self.options.loglevel.upper() + ch.setLevel(ll) + # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted) + formatter = logging.Formatter(fmt = '%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + fh.setFormatter(formatter) + ch.setFormatter(formatter) + # add the handlers to the logger + self.log.addHandler(fh) + self.log.addHandler(ch) + + if self.options.trace_rpc: + rpc_logger = logging.getLogger("BitcoinRPC") + rpc_logger.setLevel(logging.DEBUG) + rpc_handler = logging.StreamHandler(sys.stdout) + rpc_handler.setLevel(logging.DEBUG) + rpc_logger.addHandler(rpc_handler) # Test framework for doing p2p comparison testing, which sets up some bitcoind # binaries: @@ -214,6 +239,6 @@ class ComparisonTestFramework(BitcoinTestFramework): def setup_network(self): self.nodes = start_nodes( self.num_nodes, self.options.tmpdir, - extra_args=[['-debug', '-whitelist=127.0.0.1']] * self.num_nodes, + extra_args=[['-whitelist=127.0.0.1']] * self.num_nodes, binary=[self.options.testbinary] + [self.options.refbinary]*(self.num_nodes-1)) diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index aca82c8b6f..23ac324510 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -2,11 +2,7 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - - -# -# Helpful routines for regression testing -# +"""Helpful routines for regression testing.""" import os import sys @@ -19,15 +15,19 @@ import http.client import random import shutil import subprocess +import tempfile import time import re import errno +import logging from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException COVERAGE_DIR = None +logger = logging.getLogger("TestFramework.utils") + # The maximum number of nodes a single test can spawn MAX_NODES = 8 # Don't assign rpc or p2p ports lower than this @@ -240,6 +240,7 @@ def initialize_chain(test_dir, num_nodes, cachedir): break if create_cache: + logger.debug("Creating data directories from cached datadir") #find and delete old cache directories if any exist for i in range(MAX_NODES): @@ -253,11 +254,9 @@ def initialize_chain(test_dir, num_nodes, cachedir): if i > 0: args.append("-connect=127.0.0.1:"+str(p2p_port(0))) bitcoind_processes[i] = subprocess.Popen(args) - if os.getenv("PYTHON_DEBUG", ""): - print("initialize_chain: bitcoind started, waiting for RPC to come up") + logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) - if os.getenv("PYTHON_DEBUG", ""): - print("initialize_chain: RPC successfully started") + logger.debug("initialize_chain: RPC successfully started") rpcs = [] for i in range(MAX_NODES): @@ -309,42 +308,20 @@ def initialize_chain_clean(test_dir, num_nodes): datadir=initialize_datadir(test_dir, i) -def _rpchost_to_args(rpchost): - '''Convert optional IP:port spec to rpcconnect/rpcport args''' - if rpchost is None: - return [] - - match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost) - if not match: - raise ValueError('Invalid RPC host spec ' + rpchost) - - rpcconnect = match.group(1) - rpcport = match.group(2) - - if rpcconnect.startswith('['): # remove IPv6 [...] wrapping - rpcconnect = rpcconnect[1:-1] - - rv = ['-rpcconnect=' + rpcconnect] - if rpcport: - rv += ['-rpcport=' + rpcport] - return rv - -def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): +def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): """ Start a bitcoind and return RPC connection to it """ datadir = os.path.join(dirname, "node"+str(i)) if binary is None: binary = os.getenv("BITCOIND", "bitcoind") - args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-mocktime="+str(get_mocktime()) ] + args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-mocktime="+str(get_mocktime()) ] if extra_args is not None: args.extend(extra_args) - bitcoind_processes[i] = subprocess.Popen(args) - if os.getenv("PYTHON_DEBUG", ""): - print("start_node: bitcoind started, waiting for RPC to come up") + bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) + logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") url = rpc_url(i, rpchost) wait_for_bitcoind_start(bitcoind_processes[i], url, i) - if os.getenv("PYTHON_DEBUG", ""): - print("start_node: RPC successfully started") + logger.debug("initialize_chain: RPC successfully started") proxy = get_rpc_proxy(url, i, timeout=timewait) if COVERAGE_DIR: @@ -352,6 +329,25 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= return proxy +def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): + with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: + try: + node = start_node(i, dirname, extra_args, stderr=log_stderr) + stop_node(node, i) + except Exception as e: + assert 'bitcoind exited' in str(e) #node must have shutdown + if expected_msg is not None: + log_stderr.seek(0) + stderr = log_stderr.read().decode('utf-8') + if expected_msg not in stderr: + raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr) + else: + if expected_msg is None: + assert_msg = "bitcoind should have exited with an error" + else: + assert_msg = "bitcoind should have exited with expected error " + expected_msg + raise AssertionError(assert_msg) + def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): """ Start multiple bitcoinds, return RPC connections to them @@ -371,32 +367,24 @@ def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) def stop_node(node, i): + logger.debug("Stopping node %d" % i) try: node.stop() except http.client.CannotSendRequest as e: - print("WARN: Unable to stop node: " + repr(e)) - bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) + logger.exception("Unable to stop node") + return_code = bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) + assert_equal(return_code, 0) del bitcoind_processes[i] def stop_nodes(nodes): - for node in nodes: - try: - node.stop() - except http.client.CannotSendRequest as e: - print("WARN: Unable to stop node: " + repr(e)) - del nodes[:] # Emptying array closes connections as a side effect - wait_bitcoinds() + for i, node in enumerate(nodes): + stop_node(node, i) + assert not bitcoind_processes.values() # All connections must be gone now def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) -def wait_bitcoinds(): - # Wait for all bitcoinds to cleanly exit - for bitcoind in bitcoind_processes.values(): - bitcoind.wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - bitcoind_processes.clear() - def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:"+str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") @@ -455,47 +443,6 @@ def make_change(from_node, amount_in, amount_out, fee): outputs[from_node.getnewaddress()] = change return outputs -def send_zeropri_transaction(from_node, to_node, amount, fee): - """ - Create&broadcast a zero-priority transaction. - Returns (txid, hex-encoded-txdata) - Ensures transaction is zero-priority by first creating a send-to-self, - then using its output - """ - - # Create a send-to-self with confirmed inputs: - self_address = from_node.getnewaddress() - (total_in, inputs) = gather_inputs(from_node, amount+fee*2) - outputs = make_change(from_node, total_in, amount+fee, fee) - outputs[self_address] = float(amount+fee) - - self_rawtx = from_node.createrawtransaction(inputs, outputs) - self_signresult = from_node.signrawtransaction(self_rawtx) - self_txid = from_node.sendrawtransaction(self_signresult["hex"], True) - - vout = find_output(from_node, self_txid, amount+fee) - # Now immediately spend the output to create a 1-input, 1-output - # zero-priority transaction: - inputs = [ { "txid" : self_txid, "vout" : vout } ] - outputs = { to_node.getnewaddress() : float(amount) } - - rawtx = from_node.createrawtransaction(inputs, outputs) - signresult = from_node.signrawtransaction(rawtx) - txid = from_node.sendrawtransaction(signresult["hex"], True) - - return (txid, signresult["hex"]) - -def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants): - """ - Create a random zero-priority transaction. - Returns (txid, hex-encoded-transaction-data, fee) - """ - from_node = random.choice(nodes) - to_node = random.choice(nodes) - fee = min_fee + fee_increment*random.randint(0,fee_variants) - (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee) - return (txid, txhex, fee) - def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random transaction. @@ -550,13 +497,30 @@ def assert_raises_message(exc, message, fun, *args, **kwds): else: raise AssertionError("No exception raised") -def assert_raises_jsonrpc(code, fun, *args, **kwds): - '''Check for specific JSONRPC exception code''' +def assert_raises_jsonrpc(code, message, fun, *args, **kwds): + """Run an RPC and verify that a specific JSONRPC exception code and message is raised. + + Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException + and verifies that the error code and message are as expected. Throws AssertionError if + no JSONRPCException was returned or if the error code/message are not as expected. + + Args: + code (int), optional: the error code returned by the RPC call (defined + in src/rpc/protocol.h). Set to None if checking the error code is not required. + message (string), optional: [a substring of] the error string returned by the + RPC call. Set to None if checking the error string is not required + fun (function): the function to call. This should be the name of an RPC. + args*: positional arguments for the function. + kwds**: named arguments for the function. + """ try: fun(*args, **kwds) except JSONRPCException as e: - if e.error["code"] != code: + # JSONRPCException was thrown as expected. Check the code and message values are correct. + if (code is not None) and (code != e.error["code"]): raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) + if (message is not None) and (message not in e.error['message']): + raise AssertionError("Expected substring not found:"+e.error['message']) except Exception as e: raise AssertionError("Unexpected exception raised: "+type(e).__name__) else: |