diff options
Diffstat (limited to 'qa/rpc-tests/test_framework')
| -rw-r--r-- | qa/rpc-tests/test_framework/authproxy.py | 21 | ||||
| -rw-r--r-- | qa/rpc-tests/test_framework/blockstore.py | 59 | ||||
| -rw-r--r-- | qa/rpc-tests/test_framework/blocktools.py | 23 | ||||
| -rwxr-xr-x | qa/rpc-tests/test_framework/mininode.py | 14 | ||||
| -rwxr-xr-x | qa/rpc-tests/test_framework/test_framework.py | 31 | ||||
| -rw-r--r-- | qa/rpc-tests/test_framework/util.py | 70 |
6 files changed, 150 insertions, 68 deletions
diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index e5f7ab3656..d095a56ce7 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -67,9 +67,11 @@ def EncodeDecimal(o): class AuthServiceProxy(object): __id_count = 0 - def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None): + # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps + def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True): self.__service_url = service_url self._service_name = service_name + self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests self.__url = urlparse.urlparse(service_url) if self.__url.port is None: port = 80 @@ -124,17 +126,22 @@ class AuthServiceProxy(object): return self._get_response() else: raise + except BrokenPipeError: + # Python 3.5+ raises this instead of BadStatusLine when the connection was reset + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() def __call__(self, *args): AuthServiceProxy.__id_count += 1 log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal))) + json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) postdata = json.dumps({'version': '1.1', 'method': self._service_name, 'params': args, - 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal) - response = self._request('POST', self.__url.path, postdata) + 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + response = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: raise JSONRPCException(response['error']) elif 'result' not in response: @@ -144,9 +151,9 @@ class AuthServiceProxy(object): return response['result'] def _batch(self, rpc_call_list): - postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal) + postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> "+postdata) - return self._request('POST', self.__url.path, postdata) + return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): http_response = self.__conn.getresponse() @@ -162,7 +169,7 @@ class AuthServiceProxy(object): responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) if "error" in response and response["error"] is None: - log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal))) + log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- "+responsedata) return response diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py index 4bc279032b..6120dd574b 100644 --- a/qa/rpc-tests/test_framework/blockstore.py +++ b/qa/rpc-tests/test_framework/blockstore.py @@ -13,20 +13,31 @@ class BlockStore(object): self.blockDB = dbm.ndbm.open(datadir + "/blocks", 'c') self.currentBlock = 0 self.headers_map = dict() - + def close(self): self.blockDB.close() + def erase(self, blockhash): + del self.blockDB[repr(blockhash)] + + # lookup an entry and return the item as raw bytes def get(self, blockhash): - serialized_block = None + value = None try: - serialized_block = self.blockDB[repr(blockhash)] + value = self.blockDB[repr(blockhash)] except KeyError: return None - f = BytesIO(serialized_block) - ret = CBlock() - ret.deserialize(f) - ret.calc_sha256() + return value + + # lookup an entry and return it as a CBlock + def get_block(self, blockhash): + ret = None + serialized_block = self.get(blockhash) + if serialized_block is not None: + f = BytesIO(serialized_block) + ret = CBlock() + ret.deserialize(f) + ret.calc_sha256() return ret def get_header(self, blockhash): @@ -75,13 +86,16 @@ class BlockStore(object): def add_header(self, header): self.headers_map[header.sha256] = header + # lookup the hashes in "inv", and return p2p messages for delivering + # blocks found. def get_blocks(self, inv): responses = [] for i in inv: if (i.type == 2): # MSG_BLOCK - block = self.get(i.hash) - if block is not None: - responses.append(msg_block(block)) + data = self.get(i.hash) + if data is not None: + # Use msg_generic to avoid re-serialization + responses.append(msg_generic(b"block", data)) return responses def get_locator(self, current_tip=None): @@ -90,11 +104,11 @@ class BlockStore(object): r = [] counter = 0 step = 1 - lastBlock = self.get(current_tip) + lastBlock = self.get_block(current_tip) while lastBlock is not None: r.append(lastBlock.hashPrevBlock) for i in range(step): - lastBlock = self.get(lastBlock.hashPrevBlock) + lastBlock = self.get_block(lastBlock.hashPrevBlock) if lastBlock is None: break counter += 1 @@ -111,16 +125,23 @@ class TxStore(object): def close(self): self.txDB.close() + # lookup an entry and return the item as raw bytes def get(self, txhash): - serialized_tx = None + value = None try: - serialized_tx = self.txDB[repr(txhash)] + value = self.txDB[repr(txhash)] except KeyError: return None - f = BytesIO(serialized_tx) - ret = CTransaction() - ret.deserialize(f) - ret.calc_sha256() + return value + + def get_transaction(self, txhash): + ret = None + serialized_tx = self.get(txhash) + if serialized_tx is not None: + f = BytesIO(serialized_tx) + ret = CTransaction() + ret.deserialize(f) + ret.calc_sha256() return ret def add_transaction(self, tx): @@ -136,5 +157,5 @@ class TxStore(object): if (i.type == 1): # MSG_TX tx = self.get(i.hash) if tx is not None: - responses.append(msg_tx(tx)) + responses.append(msg_generic(b"tx", tx)) return responses diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py index 44232153ac..26cc396315 100644 --- a/qa/rpc-tests/test_framework/blocktools.py +++ b/qa/rpc-tests/test_framework/blocktools.py @@ -56,12 +56,27 @@ def create_coinbase(height, pubkey = None): coinbase.calc_sha256() return coinbase -# Create a transaction with an anyone-can-spend output, that spends the -# nth output of prevtx. -def create_transaction(prevtx, n, sig, value): +# Create a transaction. +# If the scriptPubKey is not specified, make it anyone-can-spend. +def create_transaction(prevtx, n, sig, value, scriptPubKey=CScript()): tx = CTransaction() assert(n < len(prevtx.vout)) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff)) - tx.vout.append(CTxOut(value, b"")) + tx.vout.append(CTxOut(value, scriptPubKey)) tx.calc_sha256() return tx + +def get_legacy_sigopcount_block(block, fAccurate=True): + count = 0 + for tx in block.vtx: + count += get_legacy_sigopcount_tx(tx, fAccurate) + return count + +def get_legacy_sigopcount_tx(tx, fAccurate=True): + count = 0 + for i in tx.vout: + count += i.scriptPubKey.GetSigOpCount(fAccurate) + for j in tx.vin: + # scriptSig might be of type bytes, so convert to CScript for the moment + count += CScript(j.scriptSig).GetSigOpCount(fAccurate) + return count diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 1617daa200..6612b99b84 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -836,6 +836,18 @@ class msg_block(object): def __repr__(self): return "msg_block(block=%s)" % (repr(self.block)) +# for cases where a user needs tighter control over what is sent over the wire +# note that the user must supply the name of the command, and the data +class msg_generic(object): + def __init__(self, command, data=None): + self.command = command + self.data = data + + def serialize(self): + return self.data + + def __repr__(self): + return "msg_generic()" class msg_getaddr(object): command = b"getaddr" @@ -1303,7 +1315,7 @@ class NodeConn(asyncore.dispatcher): def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: - return + raise IOError('Not connected, no pushbuf') self.show_debug_msg("Send %s" % repr(message)) command = message.command data = message.serialize() diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 3480de6c6e..30e8b5755d 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -15,7 +15,6 @@ import traceback from .util import ( initialize_chain, - assert_equal, start_nodes, connect_nodes_bi, sync_blocks, @@ -32,21 +31,26 @@ from .authproxy import JSONRPCException class BitcoinTestFramework(object): - # These may be over-ridden by subclasses: + def __init__(self): + self.num_nodes = 4 + self.setup_clean_chain = False + self.nodes = None + def run_test(self): - for node in self.nodes: - assert_equal(node.getblockcount(), 200) - assert_equal(node.getbalance(), 25*50) + raise NotImplementedError def add_options(self, parser): pass def setup_chain(self): print("Initializing test directory "+self.options.tmpdir) - initialize_chain(self.options.tmpdir) + if self.setup_clean_chain: + initialize_chain_clean(self.options.tmpdir, self.num_nodes) + else: + initialize_chain(self.options.tmpdir, self.num_nodes) def setup_nodes(self): - return start_nodes(4, self.options.tmpdir) + return start_nodes(self.num_nodes, self.options.tmpdir) def setup_network(self, split = False): self.nodes = self.setup_nodes() @@ -115,6 +119,8 @@ class BitcoinTestFramework(object): self.add_options(parser) (self.options, self.args) = parser.parse_args() + self.options.tmpdir += '/' + str(self.options.port_seed) + if self.options.trace_rpc: logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -161,9 +167,11 @@ class BitcoinTestFramework(object): else: print("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown: + if not self.options.nocleanup and not self.options.noshutdown and success: print("Cleaning up") shutil.rmtree(self.options.tmpdir) + else: + print("Not cleaning up dir %s" % self.options.tmpdir) if success: print("Tests successful") @@ -181,9 +189,10 @@ class BitcoinTestFramework(object): class ComparisonTestFramework(BitcoinTestFramework): - # Can override the num_nodes variable to indicate how many nodes to run. def __init__(self): + super().__init__() self.num_nodes = 2 + self.setup_clean_chain = True def add_options(self, parser): parser.add_option("--testbinary", dest="testbinary", @@ -193,10 +202,6 @@ class ComparisonTestFramework(BitcoinTestFramework): default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to use for reference nodes (if any)") - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, self.num_nodes) - def setup_network(self): self.nodes = start_nodes( self.num_nodes, self.options.tmpdir, diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index d6bb32b18c..32fe79efc3 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -35,6 +35,8 @@ PORT_MIN = 11000 # The number of ports to "reserve" for p2p and rpc, each PORT_RANGE = 5000 +BITCOIND_PROC_WAIT_TIMEOUT = 60 + class PortSeed: # Must be initialized with a unique integer for each process @@ -119,30 +121,34 @@ def hex_str_to_bytes(hex_str): def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') -def sync_blocks(rpc_connections, wait=1): +def sync_blocks(rpc_connections, wait=1, timeout=60): """ - Wait until everybody has the same block count + Wait until everybody has the same tip """ - while True: - counts = [ x.getblockcount() for x in rpc_connections ] - if counts == [ counts[0] ]*len(counts): - break + while timeout > 0: + tips = [ x.getbestblockhash() for x in rpc_connections ] + if tips == [ tips[0] ]*len(tips): + return True time.sleep(wait) + timeout -= wait + raise AssertionError("Block sync failed") -def sync_mempools(rpc_connections, wait=1): +def sync_mempools(rpc_connections, wait=1, timeout=60): """ Wait until everybody has the same transactions in their memory pools """ - while True: + while timeout > 0: pool = set(rpc_connections[0].getrawmempool()) num_match = 1 for i in range(1, len(rpc_connections)): if set(rpc_connections[i].getrawmempool()) == pool: num_match = num_match+1 if num_match == len(rpc_connections): - break + return True time.sleep(wait) + timeout -= wait + raise AssertionError("Mempool sync failed") bitcoind_processes = {} @@ -187,24 +193,28 @@ def wait_for_bitcoind_start(process, url, i): raise # unkown JSON RPC exception time.sleep(0.25) -def initialize_chain(test_dir): +def initialize_chain(test_dir, num_nodes): """ - Create (or copy from cache) a 200-block-long chain and - 4 wallets. + Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Afterward, create num_nodes copies from the cache """ - if (not os.path.isdir(os.path.join("cache","node0")) - or not os.path.isdir(os.path.join("cache","node1")) - or not os.path.isdir(os.path.join("cache","node2")) - or not os.path.isdir(os.path.join("cache","node3"))): + assert num_nodes <= MAX_NODES + create_cache = False + for i in range(MAX_NODES): + if not os.path.isdir(os.path.join('cache', 'node'+str(i))): + create_cache = True + break + + if create_cache: #find and delete old cache directories if any exist - for i in range(4): + for i in range(MAX_NODES): if os.path.isdir(os.path.join("cache","node"+str(i))): shutil.rmtree(os.path.join("cache","node"+str(i))) # Create cache directories, run bitcoinds: - for i in range(4): + for i in range(MAX_NODES): datadir=initialize_datadir("cache", i) args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ] if i > 0: @@ -217,15 +227,18 @@ def initialize_chain(test_dir): print("initialize_chain: RPC succesfully started") rpcs = [] - for i in range(4): + for i in range(MAX_NODES): try: rpcs.append(get_rpc_proxy(rpc_url(i), i)) except: sys.stderr.write("Error connecting to "+url+"\n") sys.exit(1) - # Create a 200-block-long chain; each of the 4 nodes + # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. + # Note: To preserve compatibility with older versions of + # initialize_chain, only 4 nodes will generate coins. + # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past enable_mocktime() @@ -243,13 +256,13 @@ def initialize_chain(test_dir): stop_nodes(rpcs) wait_bitcoinds() disable_mocktime() - for i in range(4): + for i in range(MAX_NODES): os.remove(log_filename("cache", i, "debug.log")) os.remove(log_filename("cache", i, "db.log")) os.remove(log_filename("cache", i, "peers.dat")) os.remove(log_filename("cache", i, "fee_estimates.dat")) - for i in range(4): + for i in range(num_nodes): from_dir = os.path.join("cache", "node"+str(i)) to_dir = os.path.join(test_dir, "node"+str(i)) shutil.copytree(from_dir, to_dir) @@ -330,7 +343,7 @@ def stop_node(node, i): node.stop() except http.client.CannotSendRequest as e: print("WARN: Unable to stop node: " + repr(e)) - bitcoind_processes[i].wait() + bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) del bitcoind_processes[i] def stop_nodes(nodes): @@ -348,7 +361,7 @@ def set_node_times(nodes, t): def wait_bitcoinds(): # Wait for all bitcoinds to cleanly exit for bitcoind in bitcoind_processes.values(): - bitcoind.wait() + bitcoind.wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) bitcoind_processes.clear() def connect_nodes(from_connection, node_num): @@ -469,6 +482,15 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): return (txid, signresult["hex"], fee) +def assert_fee_amount(fee, tx_size, fee_per_kB): + """Assert the fee was in range""" + target_fee = tx_size * fee_per_kB / 1000 + if fee < target_fee: + raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)"%(str(fee), str(target_fee))) + # allow the wallet's estimation to be at most 2 bytes off + if fee > (tx_size + 2) * fee_per_kB / 1000: + raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)"%(str(fee), str(target_fee))) + def assert_equal(thing1, thing2): if thing1 != thing2: raise AssertionError("%s != %s"%(str(thing1),str(thing2))) |