aboutsummaryrefslogtreecommitdiff
path: root/qa/rpc-tests/test_framework
diff options
context:
space:
mode:
Diffstat (limited to 'qa/rpc-tests/test_framework')
-rw-r--r--qa/rpc-tests/test_framework/address.py74
-rw-r--r--qa/rpc-tests/test_framework/authproxy.py40
-rw-r--r--qa/rpc-tests/test_framework/bignum.py5
-rw-r--r--qa/rpc-tests/test_framework/blockstore.py78
-rw-r--r--qa/rpc-tests/test_framework/blocktools.py60
-rwxr-xr-xqa/rpc-tests/test_framework/comptool.py72
-rw-r--r--qa/rpc-tests/test_framework/coverage.py9
-rw-r--r--qa/rpc-tests/test_framework/key.py25
-rwxr-xr-xqa/rpc-tests/test_framework/mininode.py1047
-rw-r--r--qa/rpc-tests/test_framework/netutil.py34
-rw-r--r--qa/rpc-tests/test_framework/script.py89
-rw-r--r--qa/rpc-tests/test_framework/siphash.py64
-rw-r--r--qa/rpc-tests/test_framework/socks5.py13
-rwxr-xr-xqa/rpc-tests/test_framework/test_framework.py76
-rw-r--r--qa/rpc-tests/test_framework/util.py397
15 files changed, 1590 insertions, 493 deletions
diff --git a/qa/rpc-tests/test_framework/address.py b/qa/rpc-tests/test_framework/address.py
new file mode 100644
index 0000000000..50b999be61
--- /dev/null
+++ b/qa/rpc-tests/test_framework/address.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# address.py
+#
+# This file encodes and decodes BASE58 P2PKH and P2SH addresses
+#
+
+from .script import hash256, hash160, sha256, CScript, OP_0
+from .util import bytes_to_hex_str, hex_str_to_bytes
+
+chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+def byte_to_base58(b, version):
+ result = ''
+ str = bytes_to_hex_str(b)
+ str = bytes_to_hex_str(chr(version).encode('latin-1')) + str
+ checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str)))
+ str += checksum[:8]
+ value = int('0x'+str,0)
+ while value > 0:
+ result = chars[value % 58] + result
+ value //= 58
+ while (str[:2] == '00'):
+ result = chars[0] + result
+ str = str[2:]
+ return result
+
+# TODO: def base58_decode
+
+def keyhash_to_p2pkh(hash, main = False):
+ assert (len(hash) == 20)
+ version = 0 if main else 111
+ return byte_to_base58(hash, version)
+
+def scripthash_to_p2sh(hash, main = False):
+ assert (len(hash) == 20)
+ version = 5 if main else 196
+ return byte_to_base58(hash, version)
+
+def key_to_p2pkh(key, main = False):
+ key = check_key(key)
+ return keyhash_to_p2pkh(hash160(key), main)
+
+def script_to_p2sh(script, main = False):
+ script = check_script(script)
+ return scripthash_to_p2sh(hash160(script), main)
+
+def key_to_p2sh_p2wpkh(key, main = False):
+ key = check_key(key)
+ p2shscript = CScript([OP_0, hash160(key)])
+ return script_to_p2sh(p2shscript, main)
+
+def script_to_p2sh_p2wsh(script, main = False):
+ script = check_script(script)
+ p2shscript = CScript([OP_0, sha256(script)])
+ return script_to_p2sh(p2shscript, main)
+
+def check_key(key):
+ if (type(key) is str):
+ key = hex_str_to_bytes(key) # Assuming this is hex string
+ if (type(key) is bytes and (len(key) == 33 or len(key) == 65)):
+ return key
+ assert(False)
+
+def check_script(script):
+ if (type(script) is str):
+ script = hex_str_to_bytes(script) # Assuming this is hex string
+ if (type(script) is bytes or type(script) is CScript):
+ return script
+ assert(False)
diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py
index fba469a0dd..fd7f32b5c6 100644
--- a/qa/rpc-tests/test_framework/authproxy.py
+++ b/qa/rpc-tests/test_framework/authproxy.py
@@ -55,21 +55,27 @@ log = logging.getLogger("BitcoinRPC")
class JSONRPCException(Exception):
def __init__(self, rpc_error):
- Exception.__init__(self)
+ try:
+ errmsg = '%(message)s (%(code)i)' % rpc_error
+ except (KeyError, TypeError):
+ errmsg = ''
+ Exception.__init__(self, errmsg)
self.error = rpc_error
def EncodeDecimal(o):
if isinstance(o, decimal.Decimal):
- return round(o, 8)
+ return str(o)
raise TypeError(repr(o) + " is not JSON serializable")
class AuthServiceProxy(object):
__id_count = 0
- def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None):
+ # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps
+ def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True):
self.__service_url = service_url
self._service_name = service_name
+ self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests
self.__url = urlparse.urlparse(service_url)
if self.__url.port is None:
port = 80
@@ -92,11 +98,10 @@ class AuthServiceProxy(object):
self.__conn = connection
elif self.__url.scheme == 'https':
self.__conn = httplib.HTTPSConnection(self.__url.hostname, port,
- None, None, False,
- timeout)
+ timeout=timeout)
else:
self.__conn = httplib.HTTPConnection(self.__url.hostname, port,
- False, timeout)
+ timeout=timeout)
def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'):
@@ -125,17 +130,23 @@ class AuthServiceProxy(object):
return self._get_response()
else:
raise
+ except (BrokenPipeError,ConnectionResetError):
+ # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset
+ # ConnectionResetError happens on FreeBSD with Python 3.4
+ self.__conn.close()
+ self.__conn.request(method, path, postdata, headers)
+ return self._get_response()
def __call__(self, *args):
AuthServiceProxy.__id_count += 1
log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name,
- json.dumps(args, default=EncodeDecimal)))
+ json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
postdata = json.dumps({'version': '1.1',
'method': self._service_name,
'params': args,
- 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal)
- response = self._request('POST', self.__url.path, postdata)
+ 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+ response = self._request('POST', self.__url.path, postdata.encode('utf-8'))
if response['error'] is not None:
raise JSONRPCException(response['error'])
elif 'result' not in response:
@@ -145,9 +156,9 @@ class AuthServiceProxy(object):
return response['result']
def _batch(self, rpc_call_list):
- postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal)
+ postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
log.debug("--> "+postdata)
- return self._request('POST', self.__url.path, postdata)
+ return self._request('POST', self.__url.path, postdata.encode('utf-8'))
def _get_response(self):
http_response = self.__conn.getresponse()
@@ -155,10 +166,15 @@ class AuthServiceProxy(object):
raise JSONRPCException({
'code': -342, 'message': 'missing HTTP response from server'})
+ content_type = http_response.getheader('Content-Type')
+ if content_type != 'application/json':
+ raise JSONRPCException({
+ 'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)})
+
responsedata = http_response.read().decode('utf8')
response = json.loads(responsedata, parse_float=decimal.Decimal)
if "error" in response and response["error"] is None:
- log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal)))
+ log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
else:
log.debug("<-- "+responsedata)
return response
diff --git a/qa/rpc-tests/test_framework/bignum.py b/qa/rpc-tests/test_framework/bignum.py
index b0c58ccd47..ef800e4d57 100644
--- a/qa/rpc-tests/test_framework/bignum.py
+++ b/qa/rpc-tests/test_framework/bignum.py
@@ -1,16 +1,15 @@
-#
+#!/usr/bin/env python3
#
# bignum.py
#
# This file is copied from python-bitcoinlib.
#
-# Distributed under the MIT/X11 software license, see the accompanying
+# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
"""Bignum routines"""
-from __future__ import absolute_import, division, print_function, unicode_literals
import struct
diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py
index b9775b477c..28a6b92b81 100644
--- a/qa/rpc-tests/test_framework/blockstore.py
+++ b/qa/rpc-tests/test_framework/blockstore.py
@@ -1,30 +1,46 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# BlockStore: a helper class that keeps a map of blocks and implements
# helper functions for responding to getheaders and getdata,
# and for constructing a getheaders message
#
-from mininode import *
-import dbm
+from .mininode import *
+from io import BytesIO
+import dbm.dumb as dbmd
class BlockStore(object):
def __init__(self, datadir):
- self.blockDB = dbm.open(datadir + "/blocks", 'c')
- self.currentBlock = 0L
+ self.blockDB = dbmd.open(datadir + "/blocks", 'c')
+ self.currentBlock = 0
self.headers_map = dict()
-
+
def close(self):
self.blockDB.close()
+ def erase(self, blockhash):
+ del self.blockDB[repr(blockhash)]
+
+ # lookup an entry and return the item as raw bytes
def get(self, blockhash):
- serialized_block = None
+ value = None
try:
- serialized_block = self.blockDB[repr(blockhash)]
+ value = self.blockDB[repr(blockhash)]
except KeyError:
return None
- f = cStringIO.StringIO(serialized_block)
- ret = CBlock()
- ret.deserialize(f)
- ret.calc_sha256()
+ return value
+
+ # lookup an entry and return it as a CBlock
+ def get_block(self, blockhash):
+ ret = None
+ serialized_block = self.get(blockhash)
+ if serialized_block is not None:
+ f = BytesIO(serialized_block)
+ ret = CBlock()
+ ret.deserialize(f)
+ ret.calc_sha256()
return ret
def get_header(self, blockhash):
@@ -66,20 +82,23 @@ class BlockStore(object):
try:
self.blockDB[repr(block.sha256)] = bytes(block.serialize())
except TypeError as e:
- print "Unexpected error: ", sys.exc_info()[0], e.args
+ print("Unexpected error: ", sys.exc_info()[0], e.args)
self.currentBlock = block.sha256
self.headers_map[block.sha256] = CBlockHeader(block)
def add_header(self, header):
self.headers_map[header.sha256] = header
+ # lookup the hashes in "inv", and return p2p messages for delivering
+ # blocks found.
def get_blocks(self, inv):
responses = []
for i in inv:
if (i.type == 2): # MSG_BLOCK
- block = self.get(i.hash)
- if block is not None:
- responses.append(msg_block(block))
+ data = self.get(i.hash)
+ if data is not None:
+ # Use msg_generic to avoid re-serialization
+ responses.append(msg_generic(b"block", data))
return responses
def get_locator(self, current_tip=None):
@@ -88,11 +107,11 @@ class BlockStore(object):
r = []
counter = 0
step = 1
- lastBlock = self.get(current_tip)
+ lastBlock = self.get_block(current_tip)
while lastBlock is not None:
r.append(lastBlock.hashPrevBlock)
for i in range(step):
- lastBlock = self.get(lastBlock.hashPrevBlock)
+ lastBlock = self.get_block(lastBlock.hashPrevBlock)
if lastBlock is None:
break
counter += 1
@@ -104,21 +123,28 @@ class BlockStore(object):
class TxStore(object):
def __init__(self, datadir):
- self.txDB = dbm.open(datadir + "/transactions", 'c')
+ self.txDB = dbmd.open(datadir + "/transactions", 'c')
def close(self):
self.txDB.close()
+ # lookup an entry and return the item as raw bytes
def get(self, txhash):
- serialized_tx = None
+ value = None
try:
- serialized_tx = self.txDB[repr(txhash)]
+ value = self.txDB[repr(txhash)]
except KeyError:
return None
- f = cStringIO.StringIO(serialized_tx)
- ret = CTransaction()
- ret.deserialize(f)
- ret.calc_sha256()
+ return value
+
+ def get_transaction(self, txhash):
+ ret = None
+ serialized_tx = self.get(txhash)
+ if serialized_tx is not None:
+ f = BytesIO(serialized_tx)
+ ret = CTransaction()
+ ret.deserialize(f)
+ ret.calc_sha256()
return ret
def add_transaction(self, tx):
@@ -126,7 +152,7 @@ class TxStore(object):
try:
self.txDB[repr(tx.sha256)] = bytes(tx.serialize())
except TypeError as e:
- print "Unexpected error: ", sys.exc_info()[0], e.args
+ print("Unexpected error: ", sys.exc_info()[0], e.args)
def get_transactions(self, inv):
responses = []
@@ -134,5 +160,5 @@ class TxStore(object):
if (i.type == 1): # MSG_TX
tx = self.get(i.hash)
if tx is not None:
- responses.append(msg_tx(tx))
+ responses.append(msg_generic(b"tx", tx))
return responses
diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py
index 59aa8c15cc..f69958823c 100644
--- a/qa/rpc-tests/test_framework/blocktools.py
+++ b/qa/rpc-tests/test_framework/blocktools.py
@@ -1,11 +1,11 @@
+#!/usr/bin/env python3
# blocktools.py - utilities for manipulating blocks and transactions
-#
-# Distributed under the MIT/X11 software license, see the accompanying
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-from mininode import *
-from script import CScript, CScriptOp, OP_TRUE, OP_CHECKSIG
+from .mininode import *
+from .script import CScript, OP_TRUE, OP_CHECKSIG, OP_RETURN
# Create a block (with regtest difficulty)
def create_block(hashprev, coinbase, nTime=None):
@@ -22,6 +22,29 @@ def create_block(hashprev, coinbase, nTime=None):
block.calc_sha256()
return block
+# From BIP141
+WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed"
+
+# According to BIP141, blocks with witness rules active must commit to the
+# hash of all in-block transactions including witness.
+def add_witness_commitment(block, nonce=0):
+ # First calculate the merkle root of the block's
+ # transactions, with witnesses.
+ witness_nonce = nonce
+ witness_root = block.calc_witness_merkle_root()
+ witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce)))
+ # witness_nonce should go to coinbase witness.
+ block.vtx[0].wit.vtxinwit = [CTxInWitness()]
+ block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(witness_nonce)]
+
+ # witness commitment is the last OP_RETURN output in coinbase
+ output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment)
+ block.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, output_data])))
+ block.vtx[0].rehash()
+ block.hashMerkleRoot = block.calc_merkle_root()
+ block.rehash()
+
+
def serialize_script_num(value):
r = bytearray(0)
if value == 0:
@@ -29,7 +52,7 @@ def serialize_script_num(value):
neg = value < 0
absvalue = -value if neg else value
while (absvalue):
- r.append(chr(absvalue & 0xff))
+ r.append(int(absvalue & 0xff))
absvalue >>= 8
if r[-1] & 0x80:
r.append(0x80 if neg else 0)
@@ -45,7 +68,7 @@ def create_coinbase(height, pubkey = None):
coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff),
ser_string(serialize_script_num(height)), 0xffffffff))
coinbaseoutput = CTxOut()
- coinbaseoutput.nValue = 50*100000000
+ coinbaseoutput.nValue = 50 * COIN
halvings = int(height/150) # regtest
coinbaseoutput.nValue >>= halvings
if (pubkey != None):
@@ -56,12 +79,27 @@ def create_coinbase(height, pubkey = None):
coinbase.calc_sha256()
return coinbase
-# Create a transaction with an anyone-can-spend output, that spends the
-# nth output of prevtx.
-def create_transaction(prevtx, n, sig, value):
+# Create a transaction.
+# If the scriptPubKey is not specified, make it anyone-can-spend.
+def create_transaction(prevtx, n, sig, value, scriptPubKey=CScript()):
tx = CTransaction()
assert(n < len(prevtx.vout))
tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff))
- tx.vout.append(CTxOut(value, ""))
+ tx.vout.append(CTxOut(value, scriptPubKey))
tx.calc_sha256()
return tx
+
+def get_legacy_sigopcount_block(block, fAccurate=True):
+ count = 0
+ for tx in block.vtx:
+ count += get_legacy_sigopcount_tx(tx, fAccurate)
+ return count
+
+def get_legacy_sigopcount_tx(tx, fAccurate=True):
+ count = 0
+ for i in tx.vout:
+ count += i.scriptPubKey.GetSigOpCount(fAccurate)
+ for j in tx.vin:
+ # scriptSig might be of type bytes, so convert to CScript for the moment
+ count += CScript(j.scriptSig).GetSigOpCount(fAccurate)
+ return count
diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py
index e0b3ce040d..7c92d3f828 100755
--- a/qa/rpc-tests/test_framework/comptool.py
+++ b/qa/rpc-tests/test_framework/comptool.py
@@ -1,12 +1,11 @@
-#!/usr/bin/env python2
-#
-# Distributed under the MIT/X11 software license, see the accompanying
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-from mininode import *
-from blockstore import BlockStore, TxStore
-from util import p2p_port
+from .mininode import *
+from .blockstore import BlockStore, TxStore
+from .util import p2p_port
'''
This is a tool for comparing two or more bitcoinds to each other
@@ -27,31 +26,32 @@ generator that returns TestInstance objects. See below for definition.
global mininode_lock
-def wait_until(predicate, attempts=float('inf'), timeout=float('inf')):
- attempt = 0
- elapsed = 0
-
- while attempt < attempts and elapsed < timeout:
- with mininode_lock:
- if predicate():
- return True
- attempt += 1
- elapsed += 0.05
- time.sleep(0.05)
-
- return False
+class RejectResult(object):
+ '''
+ Outcome that expects rejection of a transaction or block.
+ '''
+ def __init__(self, code, reason=b''):
+ self.code = code
+ self.reason = reason
+ def match(self, other):
+ if self.code != other.code:
+ return False
+ return other.reason.startswith(self.reason)
+ def __repr__(self):
+ return '%i:%s' % (self.code,self.reason or '*')
class TestNode(NodeConnCB):
def __init__(self, block_store, tx_store):
NodeConnCB.__init__(self)
- self.create_callback_map()
self.conn = None
self.bestblockhash = None
self.block_store = block_store
self.block_request_map = {}
self.tx_store = tx_store
self.tx_request_map = {}
+ self.block_reject_map = {}
+ self.tx_reject_map = {}
# When the pingmap is non-empty we're waiting for
# a response
@@ -95,6 +95,12 @@ class TestNode(NodeConnCB):
except KeyError:
raise AssertionError("Got pong for unknown ping [%s]" % repr(message))
+ def on_reject(self, conn, message):
+ if message.message == b'tx':
+ self.tx_reject_map[message.data] = RejectResult(message.code, message.reason)
+ if message.message == b'block':
+ self.block_reject_map[message.data] = RejectResult(message.code, message.reason)
+
def send_inv(self, obj):
mtype = 2 if isinstance(obj, CBlock) else 1
self.conn.send_message(msg_inv([CInv(mtype, obj.sha256)]))
@@ -172,6 +178,10 @@ class TestManager(object):
# associated NodeConn
test_node.add_connection(self.connections[-1])
+ def clear_all_connections(self):
+ self.connections = []
+ self.test_nodes = []
+
def wait_for_disconnections(self):
def disconnected():
return all(node.closed for node in self.test_nodes)
@@ -244,6 +254,15 @@ class TestManager(object):
if outcome is None:
if c.cb.bestblockhash != self.connections[0].cb.bestblockhash:
return False
+ elif isinstance(outcome, RejectResult): # Check that block was rejected w/ code
+ if c.cb.bestblockhash == blockhash:
+ return False
+ if blockhash not in c.cb.block_reject_map:
+ print('Block not in reject map: %064x' % (blockhash))
+ return False
+ if not outcome.match(c.cb.block_reject_map[blockhash]):
+ print('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash))
+ return False
elif ((c.cb.bestblockhash == blockhash) != outcome):
# print c.cb.bestblockhash, blockhash, outcome
return False
@@ -263,6 +282,15 @@ class TestManager(object):
if c.cb.lastInv != self.connections[0].cb.lastInv:
# print c.rpc.getrawmempool()
return False
+ elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code
+ if txhash in c.cb.lastInv:
+ return False
+ if txhash not in c.cb.tx_reject_map:
+ print('Tx not in reject map: %064x' % (txhash))
+ return False
+ if not outcome.match(c.cb.tx_reject_map[txhash]):
+ print('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash))
+ return False
elif ((txhash in c.cb.lastInv) != outcome):
# print c.rpc.getrawmempool(), c.cb.lastInv
return False
@@ -364,7 +392,7 @@ class TestManager(object):
if (not self.check_mempool(tx.sha256, tx_outcome)):
raise AssertionError("Mempool test failed at test %d" % test_number)
- print "Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ]
+ print("Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ])
test_number += 1
[ c.disconnect_node() for c in self.connections ]
diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py
index 50f066a850..13b33869f5 100644
--- a/qa/rpc-tests/test_framework/coverage.py
+++ b/qa/rpc-tests/test_framework/coverage.py
@@ -1,3 +1,8 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
"""
This module contains utilities for doing coverage analysis on the RPC
interface.
@@ -45,7 +50,7 @@ class AuthServiceProxyWrapper(object):
rpc_method = self.auth_service_proxy_instance._service_name
if self.coverage_logfile:
- with open(self.coverage_logfile, 'a+') as f:
+ with open(self.coverage_logfile, 'a+', encoding='utf8') as f:
f.write("%s\n" % rpc_method)
return return_val
@@ -95,7 +100,7 @@ def write_all_rpc_commands(dirname, node):
if line and not line.startswith('='):
commands.add("%s\n" % line.split()[0])
- with open(filename, 'w') as f:
+ with open(filename, 'w', encoding='utf8') as f:
f.writelines(list(commands))
return True
diff --git a/qa/rpc-tests/test_framework/key.py b/qa/rpc-tests/test_framework/key.py
index ba3038fe04..c63a15c1e0 100644
--- a/qa/rpc-tests/test_framework/key.py
+++ b/qa/rpc-tests/test_framework/key.py
@@ -75,6 +75,9 @@ ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p,
# this specifies the curve used with ECDSA.
NID_secp256k1 = 714 # from openssl/obj_mac.h
+SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141
+SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2
+
# Thx to Sam Devlin for the ctypes magic 64-bit fix.
def _check_result(val, func, args):
if val == 0:
@@ -147,7 +150,7 @@ class CECKey(object):
r = self.get_raw_ecdh_key(other_pubkey)
return kdf(r)
- def sign(self, hash):
+ def sign(self, hash, low_s = True):
# FIXME: need unit tests for below cases
if not isinstance(hash, bytes):
raise TypeError('Hash must be bytes instance; got %r' % hash.__class__)
@@ -159,7 +162,25 @@ class CECKey(object):
mb_sig = ctypes.create_string_buffer(sig_size0.value)
result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k)
assert 1 == result
- return mb_sig.raw[:sig_size0.value]
+ assert mb_sig.raw[0] == 0x30
+ assert mb_sig.raw[1] == sig_size0.value - 2
+ total_size = mb_sig.raw[1]
+ assert mb_sig.raw[2] == 2
+ r_size = mb_sig.raw[3]
+ assert mb_sig.raw[4 + r_size] == 2
+ s_size = mb_sig.raw[5 + r_size]
+ s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big')
+ if (not low_s) or s_value <= SECP256K1_ORDER_HALF:
+ return mb_sig.raw[:sig_size0.value]
+ else:
+ low_s_value = SECP256K1_ORDER - s_value
+ low_s_bytes = (low_s_value).to_bytes(33, byteorder='big')
+ while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80:
+ low_s_bytes = low_s_bytes[1:]
+ new_s_size = len(low_s_bytes)
+ new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big')
+ new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big')
+ return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes
def verify(self, hash, sig):
"""Verify a DER signature"""
diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index 64985d58e2..4d238c08d9 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -1,7 +1,12 @@
-# mininode.py - Bitcoin P2P network half-a-node
-#
-# Distributed under the MIT/X11 software license, see the accompanying
+#!/usr/bin/env python3
+# Copyright (c) 2010 ArtForz -- public domain half-a-node
+# Copyright (c) 2012 Jeff Garzik
+# Copyright (c) 2010-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# mininode.py - Bitcoin P2P network half-a-node
#
# This python code was modified from ArtForz' public domain half-a-node, as
# found in the mini-node branch of http://github.com/jgarzik/pynode.
@@ -20,25 +25,35 @@
import struct
import socket
import asyncore
-import binascii
import time
import sys
import random
-import cStringIO
+from .util import hex_str_to_bytes, bytes_to_hex_str
+from io import BytesIO
+from codecs import encode
import hashlib
from threading import RLock
from threading import Thread
import logging
import copy
+from test_framework.siphash import siphash256
BIP0031_VERSION = 60000
-MY_VERSION = 60001 # past bip-31 for ping/pong
-MY_SUBVERSION = "/python-mininode-tester:0.0.1/"
+MY_VERSION = 70014 # past bip-31 for ping/pong
+MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
MAX_INV_SZ = 50000
+MAX_BLOCK_SIZE = 1000000
+
+COIN = 100000000 # 1 btc in satoshis
+
+NODE_NETWORK = (1 << 0)
+NODE_GETUTXO = (1 << 1)
+NODE_BLOOM = (1 << 2)
+NODE_WITNESS = (1 << 3)
# Keep our own socket map for asyncore, so that we can track disconnects
-# ourselves (to workaround an issue with closing an asyncore socket when
+# ourselves (to workaround an issue with closing an asyncore socket when
# using select)
mininode_socket_map = dict()
@@ -54,12 +69,25 @@ mininode_lock = RLock()
def sha256(s):
return hashlib.new('sha256', s).digest()
+def ripemd160(s):
+ return hashlib.new('ripemd160', s).digest()
def hash256(s):
return sha256(sha256(s))
+def ser_compact_size(l):
+ r = b""
+ if l < 253:
+ r = struct.pack("B", l)
+ elif l < 0x10000:
+ r = struct.pack("<BH", 253, l)
+ elif l < 0x100000000:
+ r = struct.pack("<BI", 254, l)
+ else:
+ r = struct.pack("<BQ", 255, l)
+ return r
-def deser_string(f):
+def deser_compact_size(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
@@ -67,176 +95,130 @@ def deser_string(f):
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
- return f.read(nit)
+ return nit
+def deser_string(f):
+ nit = deser_compact_size(f)
+ return f.read(nit)
def ser_string(s):
- if len(s) < 253:
- return chr(len(s)) + s
- elif len(s) < 0x10000:
- return chr(253) + struct.pack("<H", len(s)) + s
- elif len(s) < 0x100000000L:
- return chr(254) + struct.pack("<I", len(s)) + s
- return chr(255) + struct.pack("<Q", len(s)) + s
-
+ return ser_compact_size(len(s)) + s
def deser_uint256(f):
- r = 0L
- for i in xrange(8):
+ r = 0
+ for i in range(8):
t = struct.unpack("<I", f.read(4))[0]
r += t << (i * 32)
return r
def ser_uint256(u):
- rs = ""
- for i in xrange(8):
- rs += struct.pack("<I", u & 0xFFFFFFFFL)
+ rs = b""
+ for i in range(8):
+ rs += struct.pack("<I", u & 0xFFFFFFFF)
u >>= 32
return rs
def uint256_from_str(s):
- r = 0L
+ r = 0
t = struct.unpack("<IIIIIIII", s[:32])
- for i in xrange(8):
+ for i in range(8):
r += t[i] << (i * 32)
return r
def uint256_from_compact(c):
nbytes = (c >> 24) & 0xFF
- v = (c & 0xFFFFFFL) << (8 * (nbytes - 3))
+ v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
return v
def deser_vector(f, c):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
- for i in xrange(nit):
+ for i in range(nit):
t = c()
t.deserialize(f)
r.append(t)
return r
-def ser_vector(l):
- r = ""
- if len(l) < 253:
- r = chr(len(l))
- elif len(l) < 0x10000:
- r = chr(253) + struct.pack("<H", len(l))
- elif len(l) < 0x100000000L:
- r = chr(254) + struct.pack("<I", len(l))
- else:
- r = chr(255) + struct.pack("<Q", len(l))
+# ser_function_name: Allow for an alternate serialization function on the
+# entries in the vector (we use this for serializing the vector of transactions
+# for a witness block).
+def ser_vector(l, ser_function_name=None):
+ r = ser_compact_size(len(l))
for i in l:
- r += i.serialize()
+ if ser_function_name:
+ r += getattr(i, ser_function_name)()
+ else:
+ r += i.serialize()
return r
def deser_uint256_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
- for i in xrange(nit):
+ for i in range(nit):
t = deser_uint256(f)
r.append(t)
return r
def ser_uint256_vector(l):
- r = ""
- if len(l) < 253:
- r = chr(len(l))
- elif len(l) < 0x10000:
- r = chr(253) + struct.pack("<H", len(l))
- elif len(l) < 0x100000000L:
- r = chr(254) + struct.pack("<I", len(l))
- else:
- r = chr(255) + struct.pack("<Q", len(l))
+ r = ser_compact_size(len(l))
for i in l:
r += ser_uint256(i)
return r
def deser_string_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
- for i in xrange(nit):
+ for i in range(nit):
t = deser_string(f)
r.append(t)
return r
def ser_string_vector(l):
- r = ""
- if len(l) < 253:
- r = chr(len(l))
- elif len(l) < 0x10000:
- r = chr(253) + struct.pack("<H", len(l))
- elif len(l) < 0x100000000L:
- r = chr(254) + struct.pack("<I", len(l))
- else:
- r = chr(255) + struct.pack("<Q", len(l))
+ r = ser_compact_size(len(l))
for sv in l:
r += ser_string(sv)
return r
def deser_int_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
- for i in xrange(nit):
+ for i in range(nit):
t = struct.unpack("<i", f.read(4))[0]
r.append(t)
return r
def ser_int_vector(l):
- r = ""
- if len(l) < 253:
- r = chr(len(l))
- elif len(l) < 0x10000:
- r = chr(253) + struct.pack("<H", len(l))
- elif len(l) < 0x100000000L:
- r = chr(254) + struct.pack("<I", len(l))
- else:
- r = chr(255) + struct.pack("<Q", len(l))
+ r = ser_compact_size(len(l))
for i in l:
r += struct.pack("<i", i)
return r
+# Deserialize from a hex string representation (eg from RPC)
+def FromHex(obj, hex_string):
+ obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
+ return obj
+
+# Convert a binary-serializable object to hex (eg for submission via RPC)
+def ToHex(obj):
+ return bytes_to_hex_str(obj.serialize())
# Objects that map to bitcoind objects, which can be serialized/deserialized
class CAddress(object):
def __init__(self):
self.nServices = 1
- self.pchReserved = "\x00" * 10 + "\xff" * 2
+ self.pchReserved = b"\x00" * 10 + b"\xff" * 2
self.ip = "0.0.0.0"
self.port = 0
@@ -247,7 +229,7 @@ class CAddress(object):
self.port = struct.unpack(">H", f.read(2))[0]
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<Q", self.nServices)
r += self.pchReserved
r += socket.inet_aton(self.ip)
@@ -258,14 +240,19 @@ class CAddress(object):
return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
self.ip, self.port)
+MSG_WITNESS_FLAG = 1<<30
class CInv(object):
typemap = {
0: "Error",
1: "TX",
- 2: "Block"}
+ 2: "Block",
+ 1|MSG_WITNESS_FLAG: "WitnessTx",
+ 2|MSG_WITNESS_FLAG : "WitnessBlock",
+ 4: "CompactBlock"
+ }
- def __init__(self, t=0, h=0L):
+ def __init__(self, t=0, h=0):
self.type = t
self.hash = h
@@ -274,7 +261,7 @@ class CInv(object):
self.hash = deser_uint256(f)
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<i", self.type)
r += ser_uint256(self.hash)
return r
@@ -294,7 +281,7 @@ class CBlockLocator(object):
self.vHave = deser_uint256_vector(f)
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<i", self.nVersion)
r += ser_uint256_vector(self.vHave)
return r
@@ -314,7 +301,7 @@ class COutPoint(object):
self.n = struct.unpack("<I", f.read(4))[0]
def serialize(self):
- r = ""
+ r = b""
r += ser_uint256(self.hash)
r += struct.pack("<I", self.n)
return r
@@ -324,7 +311,7 @@ class COutPoint(object):
class CTxIn(object):
- def __init__(self, outpoint=None, scriptSig="", nSequence=0):
+ def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
if outpoint is None:
self.prevout = COutPoint()
else:
@@ -339,7 +326,7 @@ class CTxIn(object):
self.nSequence = struct.unpack("<I", f.read(4))[0]
def serialize(self):
- r = ""
+ r = b""
r += self.prevout.serialize()
r += ser_string(self.scriptSig)
r += struct.pack("<I", self.nSequence)
@@ -347,12 +334,12 @@ class CTxIn(object):
def __repr__(self):
return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
- % (repr(self.prevout), binascii.hexlify(self.scriptSig),
+ % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
self.nSequence)
class CTxOut(object):
- def __init__(self, nValue=0, scriptPubKey=""):
+ def __init__(self, nValue=0, scriptPubKey=b""):
self.nValue = nValue
self.scriptPubKey = scriptPubKey
@@ -361,15 +348,75 @@ class CTxOut(object):
self.scriptPubKey = deser_string(f)
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<q", self.nValue)
r += ser_string(self.scriptPubKey)
return r
def __repr__(self):
return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
- % (self.nValue // 100000000, self.nValue % 100000000,
- binascii.hexlify(self.scriptPubKey))
+ % (self.nValue // COIN, self.nValue % COIN,
+ bytes_to_hex_str(self.scriptPubKey))
+
+
+class CScriptWitness(object):
+ def __init__(self):
+ # stack is a vector of strings
+ self.stack = []
+
+ def __repr__(self):
+ return "CScriptWitness(%s)" % \
+ (",".join([bytes_to_hex_str(x) for x in self.stack]))
+
+ def is_null(self):
+ if self.stack:
+ return False
+ return True
+
+
+class CTxInWitness(object):
+ def __init__(self):
+ self.scriptWitness = CScriptWitness()
+
+ def deserialize(self, f):
+ self.scriptWitness.stack = deser_string_vector(f)
+
+ def serialize(self):
+ return ser_string_vector(self.scriptWitness.stack)
+
+ def __repr__(self):
+ return repr(self.scriptWitness)
+
+ def is_null(self):
+ return self.scriptWitness.is_null()
+
+
+class CTxWitness(object):
+ def __init__(self):
+ self.vtxinwit = []
+
+ def deserialize(self, f):
+ for i in range(len(self.vtxinwit)):
+ self.vtxinwit[i].deserialize(f)
+
+ def serialize(self):
+ r = b""
+ # This is different than the usual vector serialization --
+ # we omit the length of the vector, which is required to be
+ # the same length as the transaction's vin vector.
+ for x in self.vtxinwit:
+ r += x.serialize()
+ return r
+
+ def __repr__(self):
+ return "CTxWitness(%s)" % \
+ (';'.join([repr(x) for x in self.vtxinwit]))
+
+ def is_null(self):
+ for x in self.vtxinwit:
+ if not x.is_null():
+ return False
+ return True
class CTransaction(object):
@@ -378,6 +425,7 @@ class CTransaction(object):
self.nVersion = 1
self.vin = []
self.vout = []
+ self.wit = CTxWitness()
self.nLockTime = 0
self.sha256 = None
self.hash = None
@@ -386,44 +434,92 @@ class CTransaction(object):
self.vin = copy.deepcopy(tx.vin)
self.vout = copy.deepcopy(tx.vout)
self.nLockTime = tx.nLockTime
- self.sha256 = None
- self.hash = None
+ self.sha256 = tx.sha256
+ self.hash = tx.hash
+ self.wit = copy.deepcopy(tx.wit)
def deserialize(self, f):
self.nVersion = struct.unpack("<i", f.read(4))[0]
self.vin = deser_vector(f, CTxIn)
- self.vout = deser_vector(f, CTxOut)
+ flags = 0
+ if len(self.vin) == 0:
+ flags = struct.unpack("<B", f.read(1))[0]
+ # Not sure why flags can't be zero, but this
+ # matches the implementation in bitcoind
+ if (flags != 0):
+ self.vin = deser_vector(f, CTxIn)
+ self.vout = deser_vector(f, CTxOut)
+ else:
+ self.vout = deser_vector(f, CTxOut)
+ if flags != 0:
+ self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
+ self.wit.deserialize(f)
self.nLockTime = struct.unpack("<I", f.read(4))[0]
self.sha256 = None
self.hash = None
- def serialize(self):
- r = ""
+ def serialize_without_witness(self):
+ r = b""
+ r += struct.pack("<i", self.nVersion)
+ r += ser_vector(self.vin)
+ r += ser_vector(self.vout)
+ r += struct.pack("<I", self.nLockTime)
+ return r
+
+ # Only serialize with witness when explicitly called for
+ def serialize_with_witness(self):
+ flags = 0
+ if not self.wit.is_null():
+ flags |= 1
+ r = b""
r += struct.pack("<i", self.nVersion)
+ if flags:
+ dummy = []
+ r += ser_vector(dummy)
+ r += struct.pack("<B", flags)
r += ser_vector(self.vin)
r += ser_vector(self.vout)
+ if flags & 1:
+ if (len(self.wit.vtxinwit) != len(self.vin)):
+ # vtxinwit must have the same length as vin
+ self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
+ for i in range(len(self.wit.vtxinwit), len(self.vin)):
+ self.wit.vtxinwit.append(CTxInWitness())
+ r += self.wit.serialize()
r += struct.pack("<I", self.nLockTime)
return r
+ # Regular serialization is without witness -- must explicitly
+ # call serialize_with_witness to include witness data.
+ def serialize(self):
+ return self.serialize_without_witness()
+
+ # Recalculate the txid (transaction hash without witness)
def rehash(self):
self.sha256 = None
self.calc_sha256()
- def calc_sha256(self):
+ # We will only cache the serialization without witness in
+ # self.sha256 and self.hash -- those are expected to be the txid.
+ def calc_sha256(self, with_witness=False):
+ if with_witness:
+ # Don't cache the result, just return it
+ return uint256_from_str(hash256(self.serialize_with_witness()))
+
if self.sha256 is None:
- self.sha256 = uint256_from_str(hash256(self.serialize()))
- self.hash = hash256(self.serialize())[::-1].encode('hex_codec')
+ self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
+ self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
def is_valid(self):
self.calc_sha256()
for tout in self.vout:
- if tout.nValue < 0 or tout.nValue > 21000000L * 100000000L:
+ if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
return False
return True
def __repr__(self):
- return "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i)" \
- % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime)
+ return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
+ % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
class CBlockHeader(object):
@@ -462,7 +558,7 @@ class CBlockHeader(object):
self.hash = None
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<i", self.nVersion)
r += ser_uint256(self.hashPrevBlock)
r += ser_uint256(self.hashMerkleRoot)
@@ -473,7 +569,7 @@ class CBlockHeader(object):
def calc_sha256(self):
if self.sha256 is None:
- r = ""
+ r = b""
r += struct.pack("<i", self.nVersion)
r += ser_uint256(self.hashPrevBlock)
r += ser_uint256(self.hashMerkleRoot)
@@ -481,7 +577,7 @@ class CBlockHeader(object):
r += struct.pack("<I", self.nBits)
r += struct.pack("<I", self.nNonce)
self.sha256 = uint256_from_str(hash256(r))
- self.hash = hash256(r)[::-1].encode('hex_codec')
+ self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
def rehash(self):
self.sha256 = None
@@ -503,25 +599,43 @@ class CBlock(CBlockHeader):
super(CBlock, self).deserialize(f)
self.vtx = deser_vector(f, CTransaction)
- def serialize(self):
- r = ""
+ def serialize(self, with_witness=False):
+ r = b""
r += super(CBlock, self).serialize()
- r += ser_vector(self.vtx)
+ if with_witness:
+ r += ser_vector(self.vtx, "serialize_with_witness")
+ else:
+ r += ser_vector(self.vtx)
return r
- def calc_merkle_root(self):
- hashes = []
- for tx in self.vtx:
- tx.calc_sha256()
- hashes.append(ser_uint256(tx.sha256))
+ # Calculate the merkle root given a vector of transaction hashes
+ def get_merkle_root(self, hashes):
while len(hashes) > 1:
newhashes = []
- for i in xrange(0, len(hashes), 2):
+ for i in range(0, len(hashes), 2):
i2 = min(i+1, len(hashes)-1)
newhashes.append(hash256(hashes[i] + hashes[i2]))
hashes = newhashes
return uint256_from_str(hashes[0])
+ def calc_merkle_root(self):
+ hashes = []
+ for tx in self.vtx:
+ tx.calc_sha256()
+ hashes.append(ser_uint256(tx.sha256))
+ return self.get_merkle_root(hashes)
+
+ def calc_witness_merkle_root(self):
+ # For witness root purposes, the hash of the
+ # coinbase, with witness, is defined to be 0...0
+ hashes = [ser_uint256(0)]
+
+ for tx in self.vtx[1:]:
+ # Calculate the hashes with witness data
+ hashes.append(ser_uint256(tx.calc_sha256(True)))
+
+ return self.get_merkle_root(hashes)
+
def is_valid(self):
self.calc_sha256()
target = uint256_from_compact(self.nBits)
@@ -535,7 +649,7 @@ class CBlock(CBlockHeader):
return True
def solve(self):
- self.calc_sha256()
+ self.rehash()
target = uint256_from_compact(self.nBits)
while self.sha256 > target:
self.nNonce += 1
@@ -559,9 +673,9 @@ class CUnsignedAlert(object):
self.nMaxVer = 0
self.setSubVer = []
self.nPriority = 0
- self.strComment = ""
- self.strStatusBar = ""
- self.strReserved = ""
+ self.strComment = b""
+ self.strStatusBar = b""
+ self.strReserved = b""
def deserialize(self, f):
self.nVersion = struct.unpack("<i", f.read(4))[0]
@@ -579,7 +693,7 @@ class CUnsignedAlert(object):
self.strReserved = deser_string(f)
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<i", self.nVersion)
r += struct.pack("<q", self.nRelayUntil)
r += struct.pack("<q", self.nExpiration)
@@ -604,15 +718,15 @@ class CUnsignedAlert(object):
class CAlert(object):
def __init__(self):
- self.vchMsg = ""
- self.vchSig = ""
+ self.vchMsg = b""
+ self.vchSig = b""
def deserialize(self, f):
self.vchMsg = deser_string(f)
self.vchSig = deser_string(f)
def serialize(self):
- r = ""
+ r = b""
r += ser_string(self.vchMsg)
r += ser_string(self.vchSig)
return r
@@ -622,14 +736,216 @@ class CAlert(object):
% (len(self.vchMsg), len(self.vchSig))
+class PrefilledTransaction(object):
+ def __init__(self, index=0, tx = None):
+ self.index = index
+ self.tx = tx
+
+ def deserialize(self, f):
+ self.index = deser_compact_size(f)
+ self.tx = CTransaction()
+ self.tx.deserialize(f)
+
+ def serialize(self, with_witness=False):
+ r = b""
+ r += ser_compact_size(self.index)
+ if with_witness:
+ r += self.tx.serialize_with_witness()
+ else:
+ r += self.tx.serialize_without_witness()
+ return r
+
+ def serialize_with_witness(self):
+ return self.serialize(with_witness=True)
+
+ def __repr__(self):
+ return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
+
+# This is what we send on the wire, in a cmpctblock message.
+class P2PHeaderAndShortIDs(object):
+ def __init__(self):
+ self.header = CBlockHeader()
+ self.nonce = 0
+ self.shortids_length = 0
+ self.shortids = []
+ self.prefilled_txn_length = 0
+ self.prefilled_txn = []
+
+ def deserialize(self, f):
+ self.header.deserialize(f)
+ self.nonce = struct.unpack("<Q", f.read(8))[0]
+ self.shortids_length = deser_compact_size(f)
+ for i in range(self.shortids_length):
+ # shortids are defined to be 6 bytes in the spec, so append
+ # two zero bytes and read it in as an 8-byte number
+ self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
+ self.prefilled_txn = deser_vector(f, PrefilledTransaction)
+ self.prefilled_txn_length = len(self.prefilled_txn)
+
+ # When using version 2 compact blocks, we must serialize with_witness.
+ def serialize(self, with_witness=False):
+ r = b""
+ r += self.header.serialize()
+ r += struct.pack("<Q", self.nonce)
+ r += ser_compact_size(self.shortids_length)
+ for x in self.shortids:
+ # We only want the first 6 bytes
+ r += struct.pack("<Q", x)[0:6]
+ if with_witness:
+ r += ser_vector(self.prefilled_txn, "serialize_with_witness")
+ else:
+ r += ser_vector(self.prefilled_txn)
+ return r
+
+ def __repr__(self):
+ return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
+
+# P2P version of the above that will use witness serialization (for compact
+# block version 2)
+class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
+ def serialize(self):
+ return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
+
+# Calculate the BIP 152-compact blocks shortid for a given transaction hash
+def calculate_shortid(k0, k1, tx_hash):
+ expected_shortid = siphash256(k0, k1, tx_hash)
+ expected_shortid &= 0x0000ffffffffffff
+ return expected_shortid
+
+# This version gets rid of the array lengths, and reinterprets the differential
+# encoding into indices that can be used for lookup.
+class HeaderAndShortIDs(object):
+ def __init__(self, p2pheaders_and_shortids = None):
+ self.header = CBlockHeader()
+ self.nonce = 0
+ self.shortids = []
+ self.prefilled_txn = []
+ self.use_witness = False
+
+ if p2pheaders_and_shortids != None:
+ self.header = p2pheaders_and_shortids.header
+ self.nonce = p2pheaders_and_shortids.nonce
+ self.shortids = p2pheaders_and_shortids.shortids
+ last_index = -1
+ for x in p2pheaders_and_shortids.prefilled_txn:
+ self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
+ last_index = self.prefilled_txn[-1].index
+
+ def to_p2p(self):
+ if self.use_witness:
+ ret = P2PHeaderAndShortWitnessIDs()
+ else:
+ ret = P2PHeaderAndShortIDs()
+ ret.header = self.header
+ ret.nonce = self.nonce
+ ret.shortids_length = len(self.shortids)
+ ret.shortids = self.shortids
+ ret.prefilled_txn_length = len(self.prefilled_txn)
+ ret.prefilled_txn = []
+ last_index = -1
+ for x in self.prefilled_txn:
+ ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
+ last_index = x.index
+ return ret
+
+ def get_siphash_keys(self):
+ header_nonce = self.header.serialize()
+ header_nonce += struct.pack("<Q", self.nonce)
+ hash_header_nonce_as_str = sha256(header_nonce)
+ key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
+ key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
+ return [ key0, key1 ]
+
+ # Version 2 compact blocks use wtxid in shortids (rather than txid)
+ def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
+ self.header = CBlockHeader(block)
+ self.nonce = nonce
+ self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
+ self.shortids = []
+ self.use_witness = use_witness
+ [k0, k1] = self.get_siphash_keys()
+ for i in range(len(block.vtx)):
+ if i not in prefill_list:
+ tx_hash = block.vtx[i].sha256
+ if use_witness:
+ tx_hash = block.vtx[i].calc_sha256(with_witness=True)
+ self.shortids.append(calculate_shortid(k0, k1, tx_hash))
+
+ def __repr__(self):
+ return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
+
+
+class BlockTransactionsRequest(object):
+
+ def __init__(self, blockhash=0, indexes = None):
+ self.blockhash = blockhash
+ self.indexes = indexes if indexes != None else []
+
+ def deserialize(self, f):
+ self.blockhash = deser_uint256(f)
+ indexes_length = deser_compact_size(f)
+ for i in range(indexes_length):
+ self.indexes.append(deser_compact_size(f))
+
+ def serialize(self):
+ r = b""
+ r += ser_uint256(self.blockhash)
+ r += ser_compact_size(len(self.indexes))
+ for x in self.indexes:
+ r += ser_compact_size(x)
+ return r
+
+ # helper to set the differentially encoded indexes from absolute ones
+ def from_absolute(self, absolute_indexes):
+ self.indexes = []
+ last_index = -1
+ for x in absolute_indexes:
+ self.indexes.append(x-last_index-1)
+ last_index = x
+
+ def to_absolute(self):
+ absolute_indexes = []
+ last_index = -1
+ for x in self.indexes:
+ absolute_indexes.append(x+last_index+1)
+ last_index = absolute_indexes[-1]
+ return absolute_indexes
+
+ def __repr__(self):
+ return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
+
+
+class BlockTransactions(object):
+
+ def __init__(self, blockhash=0, transactions = None):
+ self.blockhash = blockhash
+ self.transactions = transactions if transactions != None else []
+
+ def deserialize(self, f):
+ self.blockhash = deser_uint256(f)
+ self.transactions = deser_vector(f, CTransaction)
+
+ def serialize(self, with_witness=False):
+ r = b""
+ r += ser_uint256(self.blockhash)
+ if with_witness:
+ r += ser_vector(self.transactions, "serialize_with_witness")
+ else:
+ r += ser_vector(self.transactions)
+ return r
+
+ def __repr__(self):
+ return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
+
+
# Objects that correspond to messages on the wire
class msg_version(object):
- command = "version"
+ command = b"version"
def __init__(self):
self.nVersion = MY_VERSION
self.nServices = 1
- self.nTime = time.time()
+ self.nTime = int(time.time())
self.addrTo = CAddress()
self.addrFrom = CAddress()
self.nNonce = random.getrandbits(64)
@@ -660,7 +976,7 @@ class msg_version(object):
self.nStartingHeight = None
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<i", self.nVersion)
r += struct.pack("<Q", self.nServices)
r += struct.pack("<q", self.nTime)
@@ -679,7 +995,7 @@ class msg_version(object):
class msg_verack(object):
- command = "verack"
+ command = b"verack"
def __init__(self):
pass
@@ -688,14 +1004,14 @@ class msg_verack(object):
pass
def serialize(self):
- return ""
+ return b""
def __repr__(self):
return "msg_verack()"
class msg_addr(object):
- command = "addr"
+ command = b"addr"
def __init__(self):
self.addrs = []
@@ -711,7 +1027,7 @@ class msg_addr(object):
class msg_alert(object):
- command = "alert"
+ command = b"alert"
def __init__(self):
self.alert = CAlert()
@@ -721,7 +1037,7 @@ class msg_alert(object):
self.alert.deserialize(f)
def serialize(self):
- r = ""
+ r = b""
r += self.alert.serialize()
return r
@@ -730,7 +1046,7 @@ class msg_alert(object):
class msg_inv(object):
- command = "inv"
+ command = b"inv"
def __init__(self, inv=None):
if inv is None:
@@ -749,7 +1065,7 @@ class msg_inv(object):
class msg_getdata(object):
- command = "getdata"
+ command = b"getdata"
def __init__(self, inv=None):
self.inv = inv if inv != None else []
@@ -765,11 +1081,11 @@ class msg_getdata(object):
class msg_getblocks(object):
- command = "getblocks"
+ command = b"getblocks"
def __init__(self):
self.locator = CBlockLocator()
- self.hashstop = 0L
+ self.hashstop = 0
def deserialize(self, f):
self.locator = CBlockLocator()
@@ -777,7 +1093,7 @@ class msg_getblocks(object):
self.hashstop = deser_uint256(f)
def serialize(self):
- r = ""
+ r = b""
r += self.locator.serialize()
r += ser_uint256(self.hashstop)
return r
@@ -788,7 +1104,7 @@ class msg_getblocks(object):
class msg_tx(object):
- command = "tx"
+ command = b"tx"
def __init__(self, tx=CTransaction()):
self.tx = tx
@@ -797,14 +1113,19 @@ class msg_tx(object):
self.tx.deserialize(f)
def serialize(self):
- return self.tx.serialize()
+ return self.tx.serialize_without_witness()
def __repr__(self):
return "msg_tx(tx=%s)" % (repr(self.tx))
+class msg_witness_tx(msg_tx):
+
+ def serialize(self):
+ return self.tx.serialize_with_witness()
+
class msg_block(object):
- command = "block"
+ command = b"block"
def __init__(self, block=None):
if block is None:
@@ -821,9 +1142,27 @@ class msg_block(object):
def __repr__(self):
return "msg_block(block=%s)" % (repr(self.block))
+# for cases where a user needs tighter control over what is sent over the wire
+# note that the user must supply the name of the command, and the data
+class msg_generic(object):
+ def __init__(self, command, data=None):
+ self.command = command
+ self.data = data
+
+ def serialize(self):
+ return self.data
+
+ def __repr__(self):
+ return "msg_generic()"
+
+class msg_witness_block(msg_block):
+
+ def serialize(self):
+ r = self.block.serialize(with_witness=True)
+ return r
class msg_getaddr(object):
- command = "getaddr"
+ command = b"getaddr"
def __init__(self):
pass
@@ -832,14 +1171,14 @@ class msg_getaddr(object):
pass
def serialize(self):
- return ""
+ return b""
def __repr__(self):
return "msg_getaddr()"
class msg_ping_prebip31(object):
- command = "ping"
+ command = b"ping"
def __init__(self):
pass
@@ -848,23 +1187,23 @@ class msg_ping_prebip31(object):
pass
def serialize(self):
- return ""
+ return b""
def __repr__(self):
return "msg_ping() (pre-bip31)"
class msg_ping(object):
- command = "ping"
+ command = b"ping"
- def __init__(self, nonce=0L):
+ def __init__(self, nonce=0):
self.nonce = nonce
def deserialize(self, f):
self.nonce = struct.unpack("<Q", f.read(8))[0]
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<Q", self.nonce)
return r
@@ -873,16 +1212,16 @@ class msg_ping(object):
class msg_pong(object):
- command = "pong"
+ command = b"pong"
- def __init__(self, nonce=0L):
+ def __init__(self, nonce=0):
self.nonce = nonce
def deserialize(self, f):
self.nonce = struct.unpack("<Q", f.read(8))[0]
def serialize(self):
- r = ""
+ r = b""
r += struct.pack("<Q", self.nonce)
return r
@@ -891,7 +1230,7 @@ class msg_pong(object):
class msg_mempool(object):
- command = "mempool"
+ command = b"mempool"
def __init__(self):
pass
@@ -900,13 +1239,13 @@ class msg_mempool(object):
pass
def serialize(self):
- return ""
+ return b""
def __repr__(self):
return "msg_mempool()"
class msg_sendheaders(object):
- command = "sendheaders"
+ command = b"sendheaders"
def __init__(self):
pass
@@ -915,21 +1254,22 @@ class msg_sendheaders(object):
pass
def serialize(self):
- return ""
+ return b""
def __repr__(self):
return "msg_sendheaders()"
+
# getheaders message has
# number of entries
# vector of hashes
# hash_stop (hash of last desired block header, 0 to get as many as possible)
class msg_getheaders(object):
- command = "getheaders"
+ command = b"getheaders"
def __init__(self):
self.locator = CBlockLocator()
- self.hashstop = 0L
+ self.hashstop = 0
def deserialize(self, f):
self.locator = CBlockLocator()
@@ -937,7 +1277,7 @@ class msg_getheaders(object):
self.hashstop = deser_uint256(f)
def serialize(self):
- r = ""
+ r = b""
r += self.locator.serialize()
r += ser_uint256(self.hashstop)
return r
@@ -950,7 +1290,7 @@ class msg_getheaders(object):
# headers message has
# <count> <vector of block headers>
class msg_headers(object):
- command = "headers"
+ command = b"headers"
def __init__(self):
self.headers = []
@@ -970,26 +1310,29 @@ class msg_headers(object):
class msg_reject(object):
- command = "reject"
+ command = b"reject"
+ REJECT_MALFORMED = 1
def __init__(self):
- self.message = ""
- self.code = ""
- self.reason = ""
- self.data = 0L
+ self.message = b""
+ self.code = 0
+ self.reason = b""
+ self.data = 0
def deserialize(self, f):
self.message = deser_string(f)
self.code = struct.unpack("<B", f.read(1))[0]
self.reason = deser_string(f)
- if (self.message == "block" or self.message == "tx"):
+ if (self.code != self.REJECT_MALFORMED and
+ (self.message == b"block" or self.message == b"tx")):
self.data = deser_uint256(f)
def serialize(self):
r = ser_string(self.message)
r += struct.pack("<B", self.code)
r += ser_string(self.reason)
- if (self.message == "block" or self.message == "tx"):
+ if (self.code != self.REJECT_MALFORMED and
+ (self.message == b"block" or self.message == b"tx")):
r += ser_uint256(self.data)
return r
@@ -997,12 +1340,136 @@ class msg_reject(object):
return "msg_reject: %s %d %s [%064x]" \
% (self.message, self.code, self.reason, self.data)
+# Helper function
+def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')):
+ attempt = 0
+ elapsed = 0
+
+ while attempt < attempts and elapsed < timeout:
+ with mininode_lock:
+ if predicate():
+ return True
+ attempt += 1
+ elapsed += 0.05
+ time.sleep(0.05)
+
+ return False
+
+class msg_feefilter(object):
+ command = b"feefilter"
+
+ def __init__(self, feerate=0):
+ self.feerate = feerate
+
+ def deserialize(self, f):
+ self.feerate = struct.unpack("<Q", f.read(8))[0]
+
+ def serialize(self):
+ r = b""
+ r += struct.pack("<Q", self.feerate)
+ return r
+
+ def __repr__(self):
+ return "msg_feefilter(feerate=%08x)" % self.feerate
+
+class msg_sendcmpct(object):
+ command = b"sendcmpct"
+
+ def __init__(self):
+ self.announce = False
+ self.version = 1
+
+ def deserialize(self, f):
+ self.announce = struct.unpack("<?", f.read(1))[0]
+ self.version = struct.unpack("<Q", f.read(8))[0]
+
+ def serialize(self):
+ r = b""
+ r += struct.pack("<?", self.announce)
+ r += struct.pack("<Q", self.version)
+ return r
+
+ def __repr__(self):
+ return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
+
+class msg_cmpctblock(object):
+ command = b"cmpctblock"
+
+ def __init__(self, header_and_shortids = None):
+ self.header_and_shortids = header_and_shortids
+
+ def deserialize(self, f):
+ self.header_and_shortids = P2PHeaderAndShortIDs()
+ self.header_and_shortids.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.header_and_shortids.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
+
+class msg_getblocktxn(object):
+ command = b"getblocktxn"
+
+ def __init__(self):
+ self.block_txn_request = None
+
+ def deserialize(self, f):
+ self.block_txn_request = BlockTransactionsRequest()
+ self.block_txn_request.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.block_txn_request.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
+
+class msg_blocktxn(object):
+ command = b"blocktxn"
+
+ def __init__(self):
+ self.block_transactions = BlockTransactions()
+
+ def deserialize(self, f):
+ self.block_transactions.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.block_transactions.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
+
+class msg_witness_blocktxn(msg_blocktxn):
+ def serialize(self):
+ r = b""
+ r += self.block_transactions.serialize(with_witness=True)
+ return r
# This is what a callback should look like for NodeConn
# Reimplement the on_* functions to provide handling for events
class NodeConnCB(object):
def __init__(self):
self.verack_received = False
+ # deliver_sleep_time is helpful for debugging race conditions in p2p
+ # tests; it causes message delivery to sleep for the specified time
+ # before acquiring the global lock and delivering the next message.
+ self.deliver_sleep_time = None
+ # Remember the services our peer has advertised
+ self.peer_services = None
+
+ def set_deliver_sleep_time(self, value):
+ with mininode_lock:
+ self.deliver_sleep_time = value
+
+ def get_deliver_sleep_time(self):
+ with mininode_lock:
+ return self.deliver_sleep_time
# Spin until verack message is received from the node.
# Tests may want to use this as a signal that the test can begin.
@@ -1015,35 +1482,16 @@ class NodeConnCB(object):
return
time.sleep(0.05)
- # Derived classes should call this function once to set the message map
- # which associates the derived classes' functions to incoming messages
- def create_callback_map(self):
- self.cbmap = {
- "version": self.on_version,
- "verack": self.on_verack,
- "addr": self.on_addr,
- "alert": self.on_alert,
- "inv": self.on_inv,
- "getdata": self.on_getdata,
- "getblocks": self.on_getblocks,
- "tx": self.on_tx,
- "block": self.on_block,
- "getaddr": self.on_getaddr,
- "ping": self.on_ping,
- "pong": self.on_pong,
- "headers": self.on_headers,
- "getheaders": self.on_getheaders,
- "reject": self.on_reject,
- "mempool": self.on_mempool
- }
-
def deliver(self, conn, message):
+ deliver_sleep = self.get_deliver_sleep_time()
+ if deliver_sleep is not None:
+ time.sleep(deliver_sleep)
with mininode_lock:
try:
- self.cbmap[message.command](conn, message)
+ getattr(self, 'on_' + message.command.decode('ascii'))(conn, message)
except:
- print "ERROR delivering %s (%s)" % (repr(message),
- sys.exc_info()[0])
+ print("ERROR delivering %s (%s)" % (repr(message),
+ sys.exc_info()[0]))
def on_version(self, conn, message):
if message.nVersion >= 209:
@@ -1051,6 +1499,7 @@ class NodeConnCB(object):
conn.ver_send = min(MY_VERSION, message.nVersion)
if message.nVersion < 209:
conn.ver_recv = conn.ver_send
+ conn.nServices = message.nServices
def on_verack(self, conn, message):
conn.ver_recv = conn.ver_send
@@ -1080,43 +1529,85 @@ class NodeConnCB(object):
def on_close(self, conn): pass
def on_mempool(self, conn): pass
def on_pong(self, conn, message): pass
-
+ def on_feefilter(self, conn, message): pass
+ def on_sendheaders(self, conn, message): pass
+ def on_sendcmpct(self, conn, message): pass
+ def on_cmpctblock(self, conn, message): pass
+ def on_getblocktxn(self, conn, message): pass
+ def on_blocktxn(self, conn, message): pass
+
+# More useful callbacks and functions for NodeConnCB's which have a single NodeConn
+class SingleNodeConnCB(NodeConnCB):
+ def __init__(self):
+ NodeConnCB.__init__(self)
+ self.connection = None
+ self.ping_counter = 1
+ self.last_pong = msg_pong()
+
+ def add_connection(self, conn):
+ self.connection = conn
+
+ # Wrapper for the NodeConn's send_message function
+ def send_message(self, message):
+ self.connection.send_message(message)
+
+ def send_and_ping(self, message):
+ self.send_message(message)
+ self.sync_with_ping()
+
+ def on_pong(self, conn, message):
+ self.last_pong = message
+
+ # Sync up with the node
+ def sync_with_ping(self, timeout=30):
+ def received_pong():
+ return (self.last_pong.nonce == self.ping_counter)
+ self.send_message(msg_ping(nonce=self.ping_counter))
+ success = wait_until(received_pong, timeout=timeout)
+ self.ping_counter += 1
+ return success
# The actual NodeConn class
# This class provides an interface for a p2p connection to a specified node
class NodeConn(asyncore.dispatcher):
messagemap = {
- "version": msg_version,
- "verack": msg_verack,
- "addr": msg_addr,
- "alert": msg_alert,
- "inv": msg_inv,
- "getdata": msg_getdata,
- "getblocks": msg_getblocks,
- "tx": msg_tx,
- "block": msg_block,
- "getaddr": msg_getaddr,
- "ping": msg_ping,
- "pong": msg_pong,
- "headers": msg_headers,
- "getheaders": msg_getheaders,
- "reject": msg_reject,
- "mempool": msg_mempool
+ b"version": msg_version,
+ b"verack": msg_verack,
+ b"addr": msg_addr,
+ b"alert": msg_alert,
+ b"inv": msg_inv,
+ b"getdata": msg_getdata,
+ b"getblocks": msg_getblocks,
+ b"tx": msg_tx,
+ b"block": msg_block,
+ b"getaddr": msg_getaddr,
+ b"ping": msg_ping,
+ b"pong": msg_pong,
+ b"headers": msg_headers,
+ b"getheaders": msg_getheaders,
+ b"reject": msg_reject,
+ b"mempool": msg_mempool,
+ b"feefilter": msg_feefilter,
+ b"sendheaders": msg_sendheaders,
+ b"sendcmpct": msg_sendcmpct,
+ b"cmpctblock": msg_cmpctblock,
+ b"getblocktxn": msg_getblocktxn,
+ b"blocktxn": msg_blocktxn
}
MAGIC_BYTES = {
- "mainnet": "\xf9\xbe\xb4\xd9", # mainnet
- "testnet3": "\x0b\x11\x09\x07", # testnet3
- "regtest": "\xfa\xbf\xb5\xda" # regtest
+ "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet
+ "testnet3": b"\x0b\x11\x09\x07", # testnet3
+ "regtest": b"\xfa\xbf\xb5\xda", # regtest
}
- def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=1):
+ def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK):
asyncore.dispatcher.__init__(self, map=mininode_socket_map)
self.log = logging.getLogger("NodeConn(%s:%d)" % (dstaddr, dstport))
self.dstaddr = dstaddr
self.dstport = dstport
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
- self.sendbuf = ""
- self.recvbuf = ""
+ self.sendbuf = b""
+ self.recvbuf = b""
self.ver_send = 209
self.ver_recv = 209
self.last_sent = 0
@@ -1124,6 +1615,7 @@ class NodeConn(asyncore.dispatcher):
self.network = net
self.cb = callback
self.disconnect = False
+ self.nServices = 0
# stuff version msg into sendbuf
vt = msg_version()
@@ -1133,8 +1625,8 @@ class NodeConn(asyncore.dispatcher):
vt.addrFrom.ip = "0.0.0.0"
vt.addrFrom.port = 0
self.send_message(vt, True)
- print 'MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \
- + str(dstport)
+ print('MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \
+ + str(dstport))
try:
self.connect((dstaddr, dstport))
@@ -1153,8 +1645,8 @@ class NodeConn(asyncore.dispatcher):
self.show_debug_msg("MiniNode: Closing Connection to %s:%d... "
% (self.dstaddr, self.dstport))
self.state = "closed"
- self.recvbuf = ""
- self.sendbuf = ""
+ self.recvbuf = b""
+ self.sendbuf = b""
try:
self.close()
except:
@@ -1188,53 +1680,58 @@ class NodeConn(asyncore.dispatcher):
self.sendbuf = self.sendbuf[sent:]
def got_data(self):
- while True:
- if len(self.recvbuf) < 4:
- return
- if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
- raise ValueError("got garbage %s" % repr(self.recvbuf))
- if self.ver_recv < 209:
- if len(self.recvbuf) < 4 + 12 + 4:
- return
- command = self.recvbuf[4:4+12].split("\x00", 1)[0]
- msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
- checksum = None
- if len(self.recvbuf) < 4 + 12 + 4 + msglen:
- return
- msg = self.recvbuf[4+12+4:4+12+4+msglen]
- self.recvbuf = self.recvbuf[4+12+4+msglen:]
- else:
- if len(self.recvbuf) < 4 + 12 + 4 + 4:
- return
- command = self.recvbuf[4:4+12].split("\x00", 1)[0]
- msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
- checksum = self.recvbuf[4+12+4:4+12+4+4]
- if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
+ try:
+ while True:
+ if len(self.recvbuf) < 4:
return
- msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
- th = sha256(msg)
- h = sha256(th)
- if checksum != h[:4]:
- raise ValueError("got bad checksum " + repr(self.recvbuf))
- self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
- if command in self.messagemap:
- f = cStringIO.StringIO(msg)
- t = self.messagemap[command]()
- t.deserialize(f)
- self.got_message(t)
- else:
- self.show_debug_msg("Unknown command: '" + command + "' " +
- repr(msg))
+ if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]:
+ raise ValueError("got garbage %s" % repr(self.recvbuf))
+ if self.ver_recv < 209:
+ if len(self.recvbuf) < 4 + 12 + 4:
+ return
+ command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
+ msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
+ checksum = None
+ if len(self.recvbuf) < 4 + 12 + 4 + msglen:
+ return
+ msg = self.recvbuf[4+12+4:4+12+4+msglen]
+ self.recvbuf = self.recvbuf[4+12+4+msglen:]
+ else:
+ if len(self.recvbuf) < 4 + 12 + 4 + 4:
+ return
+ command = self.recvbuf[4:4+12].split(b"\x00", 1)[0]
+ msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0]
+ checksum = self.recvbuf[4+12+4:4+12+4+4]
+ if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen:
+ return
+ msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen]
+ th = sha256(msg)
+ h = sha256(th)
+ if checksum != h[:4]:
+ raise ValueError("got bad checksum " + repr(self.recvbuf))
+ self.recvbuf = self.recvbuf[4+12+4+4+msglen:]
+ if command in self.messagemap:
+ f = BytesIO(msg)
+ t = self.messagemap[command]()
+ t.deserialize(f)
+ self.got_message(t)
+ else:
+ self.show_debug_msg("Unknown command: '" + command + "' " +
+ repr(msg))
+ except Exception as e:
+ print('got_data:', repr(e))
+ # import traceback
+ # traceback.print_tb(sys.exc_info()[2])
def send_message(self, message, pushbuf=False):
if self.state != "connected" and not pushbuf:
- return
+ raise IOError('Not connected, no pushbuf')
self.show_debug_msg("Send %s" % repr(message))
command = message.command
data = message.serialize()
tmsg = self.MAGIC_BYTES[self.network]
tmsg += command
- tmsg += "\x00" * (12 - len(command))
+ tmsg += b"\x00" * (12 - len(command))
tmsg += struct.pack("<I", len(data))
if self.ver_send >= 209:
th = sha256(data)
@@ -1246,11 +1743,11 @@ class NodeConn(asyncore.dispatcher):
self.last_sent = time.time()
def got_message(self, message):
- if message.command == "version":
+ if message.command == b"version":
if message.nVersion <= BIP0031_VERSION:
- self.messagemap['ping'] = msg_ping_prebip31
+ self.messagemap[b'ping'] = msg_ping_prebip31
if self.last_sent + 30 * 60 < time.time():
- self.send_message(self.messagemap['ping']())
+ self.send_message(self.messagemap[b'ping']())
self.show_debug_msg("Recv %s" % repr(message))
self.cb.deliver(self, message)
diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py
index b30a88a4f7..b92a9f6e1c 100644
--- a/qa/rpc-tests/test_framework/netutil.py
+++ b/qa/rpc-tests/test_framework/netutil.py
@@ -1,16 +1,17 @@
-#!/usr/bin/env python2
-# Copyright (c) 2014 The Bitcoin Core developers
+#!/usr/bin/env python3
+# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# Linux network utilities
+
import sys
import socket
import fcntl
import struct
import array
import os
-import binascii
+from binascii import unhexlify, hexlify
# Roughly based on http://voorloopnul.com/blog/a-python-netstat-in-less-than-100-lines-of-code/ by Ricardo Pascal
STATE_ESTABLISHED = '01'
@@ -43,9 +44,9 @@ def _remove_empty(array):
def _convert_ip_port(array):
host,port = array.split(':')
# convert host from mangled-per-four-bytes form as used by kernel
- host = binascii.unhexlify(host)
+ host = unhexlify(host)
host_out = ''
- for x in range(0, len(host)/4):
+ for x in range(0, len(host) // 4):
(val,) = struct.unpack('=I', host[x*4:(x+1)*4])
host_out += '%08x' % val
@@ -57,7 +58,7 @@ def netstat(typ='tcp'):
To get pid of all network process running on system, you must run this script
as superuser
'''
- with open('/proc/net/'+typ,'r') as f:
+ with open('/proc/net/'+typ,'r',encoding='utf8') as f:
content = f.readlines()
content.pop(0)
result = []
@@ -94,7 +95,7 @@ def all_interfaces():
max_possible = 8 # initial value
while True:
bytes = max_possible * struct_size
- names = array.array('B', '\0' * bytes)
+ names = array.array('B', b'\0' * bytes)
outbytes = struct.unpack('iL', fcntl.ioctl(
s.fileno(),
0x8912, # SIOCGIFCONF
@@ -105,7 +106,7 @@ def all_interfaces():
else:
break
namestr = names.tostring()
- return [(namestr[i:i+16].split('\0', 1)[0],
+ return [(namestr[i:i+16].split(b'\0', 1)[0],
socket.inet_ntoa(namestr[i+20:i+24]))
for i in range(0, outbytes, struct_size)]
@@ -136,4 +137,19 @@ def addr_to_hex(addr):
addr = sub[0] + ([0] * nullbytes) + sub[1]
else:
raise ValueError('Could not parse address %s' % addr)
- return binascii.hexlify(bytearray(addr))
+ return hexlify(bytearray(addr)).decode('ascii')
+
+def test_ipv6_local():
+ '''
+ Check for (local) IPv6 support.
+ '''
+ import socket
+ # By using SOCK_DGRAM this will not actually make a connection, but it will
+ # fail if there is no route to IPv6 localhost.
+ have_ipv6 = True
+ try:
+ s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+ s.connect(('::1', 0))
+ except socket.error:
+ have_ipv6 = False
+ return have_ipv6
diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py
index 0a78cf6fb1..83bbf20479 100644
--- a/qa/rpc-tests/test_framework/script.py
+++ b/qa/rpc-tests/test_framework/script.py
@@ -1,20 +1,23 @@
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
#
# script.py
#
# This file is modified from python-bitcoinlib.
#
-# Distributed under the MIT/X11 software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
"""Scripts
Functionality to build scripts, as well as SignatureHash().
"""
-from __future__ import absolute_import, division, print_function, unicode_literals
-from test_framework.mininode import CTransaction, CTxOut, hash256
+from .mininode import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string
+from binascii import hexlify
+import hashlib
import sys
bchr = chr
@@ -24,10 +27,9 @@ if sys.version > '3':
bchr = lambda x: bytes([x])
bord = lambda x: x
-import copy
import struct
-from test_framework.bignum import bn2vch
+from .bignum import bn2vch
MAX_SCRIPT_SIZE = 10000
MAX_SCRIPT_ELEMENT_SIZE = 520
@@ -35,6 +37,10 @@ MAX_SCRIPT_OPCODES = 201
OPCODE_NAMES = {}
+def hash160(s):
+ return hashlib.new('ripemd160', sha256(s)).digest()
+
+
_opcode_instances = []
class CScriptOp(int):
"""A single script opcode"""
@@ -226,8 +232,8 @@ OP_CHECKMULTISIGVERIFY = CScriptOp(0xaf)
# expansion
OP_NOP1 = CScriptOp(0xb0)
-OP_NOP2 = CScriptOp(0xb1)
-OP_NOP3 = CScriptOp(0xb2)
+OP_CHECKLOCKTIMEVERIFY = CScriptOp(0xb1)
+OP_CHECKSEQUENCEVERIFY = CScriptOp(0xb2)
OP_NOP4 = CScriptOp(0xb3)
OP_NOP5 = CScriptOp(0xb4)
OP_NOP6 = CScriptOp(0xb5)
@@ -353,8 +359,8 @@ VALID_OPCODES = {
OP_CHECKMULTISIGVERIFY,
OP_NOP1,
- OP_NOP2,
- OP_NOP3,
+ OP_CHECKLOCKTIMEVERIFY,
+ OP_CHECKSEQUENCEVERIFY,
OP_NOP4,
OP_NOP5,
OP_NOP6,
@@ -472,8 +478,8 @@ OPCODE_NAMES.update({
OP_CHECKMULTISIG : 'OP_CHECKMULTISIG',
OP_CHECKMULTISIGVERIFY : 'OP_CHECKMULTISIGVERIFY',
OP_NOP1 : 'OP_NOP1',
- OP_NOP2 : 'OP_NOP2',
- OP_NOP3 : 'OP_NOP3',
+ OP_CHECKLOCKTIMEVERIFY : 'OP_CHECKLOCKTIMEVERIFY',
+ OP_CHECKSEQUENCEVERIFY : 'OP_CHECKSEQUENCEVERIFY',
OP_NOP4 : 'OP_NOP4',
OP_NOP5 : 'OP_NOP5',
OP_NOP6 : 'OP_NOP6',
@@ -591,8 +597,8 @@ OPCODES_BY_NAME = {
'OP_CHECKMULTISIG' : OP_CHECKMULTISIG,
'OP_CHECKMULTISIGVERIFY' : OP_CHECKMULTISIGVERIFY,
'OP_NOP1' : OP_NOP1,
- 'OP_NOP2' : OP_NOP2,
- 'OP_NOP3' : OP_NOP3,
+ 'OP_CHECKLOCKTIMEVERIFY' : OP_CHECKLOCKTIMEVERIFY,
+ 'OP_CHECKSEQUENCEVERIFY' : OP_CHECKSEQUENCEVERIFY,
'OP_NOP4' : OP_NOP4,
'OP_NOP5' : OP_NOP5,
'OP_NOP6' : OP_NOP6,
@@ -629,7 +635,7 @@ class CScriptNum(object):
neg = obj.value < 0
absvalue = -obj.value if neg else obj.value
while (absvalue):
- r.append(chr(absvalue & 0xff))
+ r.append(absvalue & 0xff)
absvalue >>= 8
if r[-1] & 0x80:
r.append(0x80 if neg else 0)
@@ -658,7 +664,7 @@ class CScript(bytes):
other = bchr(CScriptOp(OP_0))
else:
other = CScriptNum.encode(other)
- elif isinstance(other, (int, long)):
+ elif isinstance(other, int):
if 0 <= other <= 16:
other = bytes(bchr(CScriptOp.encode_op_n(other)))
elif other == -1:
@@ -777,7 +783,7 @@ class CScript(bytes):
# need to change
def _repr(o):
if isinstance(o, bytes):
- return "x('%s')" % binascii.hexlify(o).decode('utf8')
+ return b"x('%s')" % hexlify(o).decode('ascii')
else:
return repr(o)
@@ -876,7 +882,7 @@ def SignatureHash(script, txTo, inIdx, hashtype):
tmp = txtmp.vout[outIdx]
txtmp.vout = []
for i in range(outIdx):
- txtmp.vout.append(CTxOut())
+ txtmp.vout.append(CTxOut(-1))
txtmp.vout.append(tmp)
for i in range(len(txtmp.vin)):
@@ -894,3 +900,48 @@ def SignatureHash(script, txTo, inIdx, hashtype):
hash = hash256(s)
return (hash, None)
+
+# TODO: Allow cached hashPrevouts/hashSequence/hashOutputs to be provided.
+# Performance optimization probably not necessary for python tests, however.
+# Note that this corresponds to sigversion == 1 in EvalScript, which is used
+# for version 0 witnesses.
+def SegwitVersion1SignatureHash(script, txTo, inIdx, hashtype, amount):
+
+ hashPrevouts = 0
+ hashSequence = 0
+ hashOutputs = 0
+
+ if not (hashtype & SIGHASH_ANYONECANPAY):
+ serialize_prevouts = bytes()
+ for i in txTo.vin:
+ serialize_prevouts += i.prevout.serialize()
+ hashPrevouts = uint256_from_str(hash256(serialize_prevouts))
+
+ if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+ serialize_sequence = bytes()
+ for i in txTo.vin:
+ serialize_sequence += struct.pack("<I", i.nSequence)
+ hashSequence = uint256_from_str(hash256(serialize_sequence))
+
+ if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+ serialize_outputs = bytes()
+ for o in txTo.vout:
+ serialize_outputs += o.serialize()
+ hashOutputs = uint256_from_str(hash256(serialize_outputs))
+ elif ((hashtype & 0x1f) == SIGHASH_SINGLE and inIdx < len(txTo.vout)):
+ serialize_outputs = txTo.vout[inIdx].serialize()
+ hashOutputs = uint256_from_str(hash256(serialize_outputs))
+
+ ss = bytes()
+ ss += struct.pack("<i", txTo.nVersion)
+ ss += ser_uint256(hashPrevouts)
+ ss += ser_uint256(hashSequence)
+ ss += txTo.vin[inIdx].prevout.serialize()
+ ss += ser_string(script)
+ ss += struct.pack("<q", amount)
+ ss += struct.pack("<I", txTo.vin[inIdx].nSequence)
+ ss += ser_uint256(hashOutputs)
+ ss += struct.pack("<i", txTo.nLockTime)
+ ss += struct.pack("<I", hashtype)
+
+ return hash256(ss)
diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py
new file mode 100644
index 0000000000..9c0574bd93
--- /dev/null
+++ b/qa/rpc-tests/test_framework/siphash.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# siphash.py - Specialized SipHash-2-4 implementations
+#
+# This implements SipHash-2-4 for 256-bit integers.
+
+def rotl64(n, b):
+ return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
+
+def siphash_round(v0, v1, v2, v3):
+ v0 = (v0 + v1) & ((1 << 64) - 1)
+ v1 = rotl64(v1, 13)
+ v1 ^= v0
+ v0 = rotl64(v0, 32)
+ v2 = (v2 + v3) & ((1 << 64) - 1)
+ v3 = rotl64(v3, 16)
+ v3 ^= v2
+ v0 = (v0 + v3) & ((1 << 64) - 1)
+ v3 = rotl64(v3, 21)
+ v3 ^= v0
+ v2 = (v2 + v1) & ((1 << 64) - 1)
+ v1 = rotl64(v1, 17)
+ v1 ^= v2
+ v2 = rotl64(v2, 32)
+ return (v0, v1, v2, v3)
+
+def siphash256(k0, k1, h):
+ n0 = h & ((1 << 64) - 1)
+ n1 = (h >> 64) & ((1 << 64) - 1)
+ n2 = (h >> 128) & ((1 << 64) - 1)
+ n3 = (h >> 192) & ((1 << 64) - 1)
+ v0 = 0x736f6d6570736575 ^ k0
+ v1 = 0x646f72616e646f6d ^ k1
+ v2 = 0x6c7967656e657261 ^ k0
+ v3 = 0x7465646279746573 ^ k1 ^ n0
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n0
+ v3 ^= n1
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n1
+ v3 ^= n2
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n2
+ v3 ^= n3
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n3
+ v3 ^= 0x2000000000000000
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= 0x2000000000000000
+ v2 ^= 0xFF
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ return v0 ^ v1 ^ v2 ^ v3
diff --git a/qa/rpc-tests/test_framework/socks5.py b/qa/rpc-tests/test_framework/socks5.py
index 1dbfb98d5d..372f5ed605 100644
--- a/qa/rpc-tests/test_framework/socks5.py
+++ b/qa/rpc-tests/test_framework/socks5.py
@@ -1,11 +1,12 @@
-# Copyright (c) 2015 The Bitcoin Core developers
+#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
Dummy Socks5 server for testing.
'''
-from __future__ import print_function, division, unicode_literals
-import socket, threading, Queue
+
+import socket, threading, queue
import traceback, sys
### Protocol constants
@@ -102,7 +103,7 @@ class Socks5Connection(object):
addr = recvall(self.conn, 4)
elif atyp == AddressType.DOMAINNAME:
n = recvall(self.conn, 1)[0]
- addr = str(recvall(self.conn, n))
+ addr = recvall(self.conn, n)
elif atyp == AddressType.IPV6:
addr = recvall(self.conn, 16)
else:
@@ -117,7 +118,7 @@ class Socks5Connection(object):
self.serv.queue.put(cmdin)
print('Proxy: ', cmdin)
# Fall through to disconnect
- except Exception,e:
+ except Exception as e:
traceback.print_exc(file=sys.stderr)
self.serv.queue.put(e)
finally:
@@ -132,7 +133,7 @@ class Socks5Server(object):
self.s.listen(5)
self.running = False
self.thread = None
- self.queue = Queue.Queue() # report connections and exceptions to client
+ self.queue = queue.Queue() # report connections and exceptions to client
def run(self):
while self.running:
diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py
index ae2d91ab60..e6fc5fd8a2 100755
--- a/qa/rpc-tests/test_framework/test_framework.py
+++ b/qa/rpc-tests/test_framework/test_framework.py
@@ -1,51 +1,59 @@
-#!/usr/bin/env python2
-# Copyright (c) 2014 The Bitcoin Core developers
+#!/usr/bin/env python3
+# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# Base class for RPC testing
-# Add python-bitcoinrpc to module search path:
+import logging
+import optparse
import os
import sys
-
import shutil
import tempfile
import traceback
from .util import (
initialize_chain,
- assert_equal,
start_nodes,
connect_nodes_bi,
sync_blocks,
sync_mempools,
stop_nodes,
- wait_bitcoinds,
+ stop_node,
enable_coverage,
check_json_precision,
initialize_chain_clean,
+ PortSeed,
)
-from authproxy import AuthServiceProxy, JSONRPCException
+from .authproxy import JSONRPCException
class BitcoinTestFramework(object):
- # These may be over-ridden by subclasses:
+ def __init__(self):
+ self.num_nodes = 4
+ self.setup_clean_chain = False
+ self.nodes = None
+
def run_test(self):
- for node in self.nodes:
- assert_equal(node.getblockcount(), 200)
- assert_equal(node.getbalance(), 25*50)
+ raise NotImplementedError
def add_options(self, parser):
pass
def setup_chain(self):
print("Initializing test directory "+self.options.tmpdir)
- initialize_chain(self.options.tmpdir)
+ if self.setup_clean_chain:
+ initialize_chain_clean(self.options.tmpdir, self.num_nodes)
+ else:
+ initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir)
+
+ def stop_node(self, num_node):
+ stop_node(self.nodes[num_node], num_node)
def setup_nodes(self):
- return start_nodes(4, self.options.tmpdir)
+ return start_nodes(self.num_nodes, self.options.tmpdir)
def setup_network(self, split = False):
self.nodes = self.setup_nodes()
@@ -72,7 +80,6 @@ class BitcoinTestFramework(object):
"""
assert not self.is_network_split
stop_nodes(self.nodes)
- wait_bitcoinds()
self.setup_network(True)
def sync_all(self):
@@ -91,36 +98,42 @@ class BitcoinTestFramework(object):
"""
assert self.is_network_split
stop_nodes(self.nodes)
- wait_bitcoinds()
self.setup_network(False)
def main(self):
- import optparse
parser = optparse.OptionParser(usage="%prog [options]")
parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
help="Leave bitcoinds and test.* datadir on exit or error")
parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true",
help="Don't stop bitcoinds after the test execution")
- parser.add_option("--srcdir", dest="srcdir", default="../../src",
+ parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"),
help="Source directory containing bitcoind/bitcoin-cli (default: %default)")
+ parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"),
+ help="Directory for caching pregenerated datadirs")
parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
help="Root directory for datadirs")
parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true",
help="Print out all RPC calls as they are made")
+ parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int',
+ help="The seed to use for assigning port numbers (default: current process id)")
parser.add_option("--coveragedir", dest="coveragedir",
help="Write tested RPC commands into this directory")
self.add_options(parser)
(self.options, self.args) = parser.parse_args()
+ # backup dir variable for removal at cleanup
+ self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed)
+
if self.options.trace_rpc:
- import logging
- logging.basicConfig(level=logging.DEBUG)
+ logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
if self.options.coveragedir:
enable_coverage(self.options.coveragedir)
- os.environ['PATH'] = self.options.srcdir+":"+os.environ['PATH']
+ PortSeed.n = self.options.port_seed
+
+ os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH']
check_json_precision()
@@ -140,22 +153,30 @@ class BitcoinTestFramework(object):
print("JSONRPC error: "+e.error['message'])
traceback.print_tb(sys.exc_info()[2])
except AssertionError as e:
- print("Assertion failed: "+e.message)
+ print("Assertion failed: " + str(e))
+ traceback.print_tb(sys.exc_info()[2])
+ except KeyError as e:
+ print("key not found: "+ str(e))
traceback.print_tb(sys.exc_info()[2])
except Exception as e:
- print("Unexpected exception caught during testing: "+str(e))
+ print("Unexpected exception caught during testing: " + repr(e))
traceback.print_tb(sys.exc_info()[2])
+ except KeyboardInterrupt as e:
+ print("Exiting after " + repr(e))
if not self.options.noshutdown:
print("Stopping nodes")
stop_nodes(self.nodes)
- wait_bitcoinds()
else:
print("Note: bitcoinds were not stopped and may still be running")
- if not self.options.nocleanup and not self.options.noshutdown:
+ if not self.options.nocleanup and not self.options.noshutdown and success:
print("Cleaning up")
shutil.rmtree(self.options.tmpdir)
+ if not os.listdir(self.options.root):
+ os.rmdir(self.options.root)
+ else:
+ print("Not cleaning up dir %s" % self.options.tmpdir)
if success:
print("Tests successful")
@@ -173,9 +194,10 @@ class BitcoinTestFramework(object):
class ComparisonTestFramework(BitcoinTestFramework):
- # Can override the num_nodes variable to indicate how many nodes to run.
def __init__(self):
+ super().__init__()
self.num_nodes = 2
+ self.setup_clean_chain = True
def add_options(self, parser):
parser.add_option("--testbinary", dest="testbinary",
@@ -185,10 +207,6 @@ class ComparisonTestFramework(BitcoinTestFramework):
default=os.getenv("BITCOIND", "bitcoind"),
help="bitcoind binary to use for reference nodes (if any)")
- def setup_chain(self):
- print "Initializing test directory "+self.options.tmpdir
- initialize_chain_clean(self.options.tmpdir, self.num_nodes)
-
def setup_network(self):
self.nodes = start_nodes(
self.num_nodes, self.options.tmpdir,
diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py
index b7e90a8a8b..c6b0367b41 100644
--- a/qa/rpc-tests/test_framework/util.py
+++ b/qa/rpc-tests/test_framework/util.py
@@ -1,27 +1,67 @@
-# Copyright (c) 2014 The Bitcoin Core developers
+#!/usr/bin/env python3
+# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+
#
# Helpful routines for regression testing
#
-# Add python-bitcoinrpc to module search path:
import os
import sys
+from binascii import hexlify, unhexlify
+from base64 import b64encode
from decimal import Decimal, ROUND_DOWN
import json
+import http.client
import random
import shutil
import subprocess
import time
import re
+import errno
from . import coverage
from .authproxy import AuthServiceProxy, JSONRPCException
COVERAGE_DIR = None
+# The maximum number of nodes a single test can spawn
+MAX_NODES = 8
+# Don't assign rpc or p2p ports lower than this
+PORT_MIN = 11000
+# The number of ports to "reserve" for p2p and rpc, each
+PORT_RANGE = 5000
+
+BITCOIND_PROC_WAIT_TIMEOUT = 60
+
+
+class PortSeed:
+ # Must be initialized with a unique integer for each process
+ n = None
+
+#Set Mocktime default to OFF.
+#MOCKTIME is only needed for scripts that use the
+#cached version of the blockchain. If the cached
+#version of the blockchain is used without MOCKTIME
+#then the mempools will not sync due to IBD.
+MOCKTIME = 0
+
+def enable_mocktime():
+ #For backwared compatibility of the python scripts
+ #with previous versions of the cache, set MOCKTIME
+ #to Jan 1, 2014 + (201 * 10 * 60)
+ global MOCKTIME
+ MOCKTIME = 1388534400 + (201 * 10 * 60)
+
+def disable_mocktime():
+ global MOCKTIME
+ MOCKTIME = 0
+
+def get_mocktime():
+ return MOCKTIME
def enable_coverage(dirname):
"""Maintain a log of which RPC calls are made during testing."""
@@ -56,9 +96,11 @@ def get_rpc_proxy(url, node_number, timeout=None):
def p2p_port(n):
- return 11000 + n + os.getpid()%999
+ assert(n <= MAX_NODES)
+ return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES)
+
def rpc_port(n):
- return 12000 + n + os.getpid()%999
+ return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES)
def check_json_precision():
"""Make sure json library being used does not lose precision converting BTC values"""
@@ -70,30 +112,47 @@ def check_json_precision():
def count_bytes(hex_string):
return len(bytearray.fromhex(hex_string))
-def sync_blocks(rpc_connections, wait=1):
+def bytes_to_hex_str(byte_str):
+ return hexlify(byte_str).decode('ascii')
+
+def hex_str_to_bytes(hex_str):
+ return unhexlify(hex_str.encode('ascii'))
+
+def str_to_b64str(string):
+ return b64encode(string.encode('utf-8')).decode('ascii')
+
+def sync_blocks(rpc_connections, wait=1, timeout=60):
"""
- Wait until everybody has the same block count
+ Wait until everybody has the same tip
"""
- while True:
- counts = [ x.getblockcount() for x in rpc_connections ]
- if counts == [ counts[0] ]*len(counts):
- break
- time.sleep(wait)
-
-def sync_mempools(rpc_connections, wait=1):
+ maxheight = 0
+ while timeout > 0:
+ tips = [ x.waitforblockheight(maxheight, int(wait * 1000)) for x in rpc_connections ]
+ heights = [ x["height"] for x in tips ]
+ if tips == [ tips[0] ]*len(tips):
+ return True
+ if heights == [ heights[0] ]*len(heights): #heights are the same but hashes are not
+ raise AssertionError("Block sync failed")
+ timeout -= wait
+ maxheight = max(heights)
+ raise AssertionError("Block sync failed")
+
+def sync_mempools(rpc_connections, wait=1, timeout=60):
"""
Wait until everybody has the same transactions in their memory
pools
"""
- while True:
+ while timeout > 0:
pool = set(rpc_connections[0].getrawmempool())
num_match = 1
for i in range(1, len(rpc_connections)):
if set(rpc_connections[i].getrawmempool()) == pool:
num_match = num_match+1
if num_match == len(rpc_connections):
- break
+ return True
time.sleep(wait)
+ timeout -= wait
+ raise AssertionError("Mempool sync failed")
bitcoind_processes = {}
@@ -101,62 +160,101 @@ def initialize_datadir(dirname, n):
datadir = os.path.join(dirname, "node"+str(n))
if not os.path.isdir(datadir):
os.makedirs(datadir)
- with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f:
- f.write("regtest=1\n");
- f.write("rpcuser=rt\n");
- f.write("rpcpassword=rt\n");
- f.write("port="+str(p2p_port(n))+"\n");
- f.write("rpcport="+str(rpc_port(n))+"\n");
+ rpc_u, rpc_p = rpc_auth_pair(n)
+ with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f:
+ f.write("regtest=1\n")
+ f.write("rpcuser=" + rpc_u + "\n")
+ f.write("rpcpassword=" + rpc_p + "\n")
+ f.write("port="+str(p2p_port(n))+"\n")
+ f.write("rpcport="+str(rpc_port(n))+"\n")
+ f.write("listenonion=0\n")
return datadir
-def initialize_chain(test_dir):
+def rpc_auth_pair(n):
+ return 'rpcuser💻' + str(n), 'rpcpass🔑' + str(n)
+
+def rpc_url(i, rpchost=None):
+ rpc_u, rpc_p = rpc_auth_pair(i)
+ host = '127.0.0.1'
+ port = rpc_port(i)
+ if rpchost:
+ parts = rpchost.split(':')
+ if len(parts) == 2:
+ host, port = parts
+ else:
+ host = rpchost
+ return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port))
+
+def wait_for_bitcoind_start(process, url, i):
+ '''
+ Wait for bitcoind to start. This means that RPC is accessible and fully initialized.
+ Raise an exception if bitcoind exits during initialization.
+ '''
+ while True:
+ if process.poll() is not None:
+ raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
+ try:
+ rpc = get_rpc_proxy(url, i)
+ blocks = rpc.getblockcount()
+ break # break out of loop on success
+ except IOError as e:
+ if e.errno != errno.ECONNREFUSED: # Port not yet open?
+ raise # unknown IO error
+ except JSONRPCException as e: # Initialization phase
+ if e.error['code'] != -28: # RPC in warmup?
+ raise # unkown JSON RPC exception
+ time.sleep(0.25)
+
+def initialize_chain(test_dir, num_nodes, cachedir):
"""
- Create (or copy from cache) a 200-block-long chain and
- 4 wallets.
- bitcoind and bitcoin-cli must be in search path.
+ Create a cache of a 200-block-long chain (with wallet) for MAX_NODES
+ Afterward, create num_nodes copies from the cache
"""
- if (not os.path.isdir(os.path.join("cache","node0"))
- or not os.path.isdir(os.path.join("cache","node1"))
- or not os.path.isdir(os.path.join("cache","node2"))
- or not os.path.isdir(os.path.join("cache","node3"))):
+ assert num_nodes <= MAX_NODES
+ create_cache = False
+ for i in range(MAX_NODES):
+ if not os.path.isdir(os.path.join(cachedir, 'node'+str(i))):
+ create_cache = True
+ break
+
+ if create_cache:
#find and delete old cache directories if any exist
- for i in range(4):
- if os.path.isdir(os.path.join("cache","node"+str(i))):
- shutil.rmtree(os.path.join("cache","node"+str(i)))
+ for i in range(MAX_NODES):
+ if os.path.isdir(os.path.join(cachedir,"node"+str(i))):
+ shutil.rmtree(os.path.join(cachedir,"node"+str(i)))
- devnull = open(os.devnull, "w")
# Create cache directories, run bitcoinds:
- for i in range(4):
- datadir=initialize_datadir("cache", i)
- args = [ os.getenv("BITCOIND", "bitcoind"), "-keypool=1", "-datadir="+datadir, "-discover=0" ]
+ for i in range(MAX_NODES):
+ datadir=initialize_datadir(cachedir, i)
+ args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ]
if i > 0:
args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
bitcoind_processes[i] = subprocess.Popen(args)
if os.getenv("PYTHON_DEBUG", ""):
- print "initialize_chain: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
- subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir,
- "-rpcwait", "getblockcount"], stdout=devnull)
+ print("initialize_chain: bitcoind started, waiting for RPC to come up")
+ wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i)
if os.getenv("PYTHON_DEBUG", ""):
- print "initialize_chain: bitcoin-cli -rpcwait getblockcount completed"
- devnull.close()
+ print("initialize_chain: RPC succesfully started")
rpcs = []
-
- for i in range(4):
+ for i in range(MAX_NODES):
try:
- url = "http://rt:rt@127.0.0.1:%d" % (rpc_port(i),)
- rpcs.append(get_rpc_proxy(url, i))
+ rpcs.append(get_rpc_proxy(rpc_url(i), i))
except:
sys.stderr.write("Error connecting to "+url+"\n")
sys.exit(1)
- # Create a 200-block-long chain; each of the 4 nodes
+ # Create a 200-block-long chain; each of the 4 first nodes
# gets 25 mature blocks and 25 immature.
- # blocks are created with timestamps 10 minutes apart, starting
- # at 1 Jan 2014
- block_time = 1388534400
+ # Note: To preserve compatibility with older versions of
+ # initialize_chain, only 4 nodes will generate coins.
+ #
+ # blocks are created with timestamps 10 minutes apart
+ # starting from 2010 minutes in the past
+ enable_mocktime()
+ block_time = get_mocktime() - (201 * 10 * 60)
for i in range(2):
for peer in range(4):
for j in range(25):
@@ -168,15 +266,15 @@ def initialize_chain(test_dir):
# Shut them down, and clean up cache directories:
stop_nodes(rpcs)
- wait_bitcoinds()
- for i in range(4):
- os.remove(log_filename("cache", i, "debug.log"))
- os.remove(log_filename("cache", i, "db.log"))
- os.remove(log_filename("cache", i, "peers.dat"))
- os.remove(log_filename("cache", i, "fee_estimates.dat"))
-
- for i in range(4):
- from_dir = os.path.join("cache", "node"+str(i))
+ disable_mocktime()
+ for i in range(MAX_NODES):
+ os.remove(log_filename(cachedir, i, "debug.log"))
+ os.remove(log_filename(cachedir, i, "db.log"))
+ os.remove(log_filename(cachedir, i, "peers.dat"))
+ os.remove(log_filename(cachedir, i, "fee_estimates.dat"))
+
+ for i in range(num_nodes):
+ from_dir = os.path.join(cachedir, "node"+str(i))
to_dir = os.path.join(test_dir, "node"+str(i))
shutil.copytree(from_dir, to_dir)
initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf
@@ -217,21 +315,15 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
datadir = os.path.join(dirname, "node"+str(i))
if binary is None:
binary = os.getenv("BITCOIND", "bitcoind")
- # RPC tests still depend on free transactions
- args = [ binary, "-datadir="+datadir, "-keypool=1", "-discover=0", "-rest", "-blockprioritysize=50000" ]
+ args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-mocktime="+str(get_mocktime()) ]
if extra_args is not None: args.extend(extra_args)
bitcoind_processes[i] = subprocess.Popen(args)
- devnull = open(os.devnull, "w")
if os.getenv("PYTHON_DEBUG", ""):
- print "start_node: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
- subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir] +
- _rpchost_to_args(rpchost) +
- ["-rpcwait", "getblockcount"], stdout=devnull)
+ print("start_node: bitcoind started, waiting for RPC to come up")
+ url = rpc_url(i, rpchost)
+ wait_for_bitcoind_start(bitcoind_processes[i], url, i)
if os.getenv("PYTHON_DEBUG", ""):
- print "start_node: calling bitcoin-cli -rpcwait getblockcount returned"
- devnull.close()
- url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
-
+ print("start_node: RPC succesfully started")
proxy = get_rpc_proxy(url, i, timeout=timewait)
if COVERAGE_DIR:
@@ -243,22 +335,36 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
"""
Start multiple bitcoinds, return RPC connections to them
"""
- if extra_args is None: extra_args = [ None for i in range(num_nodes) ]
- if binary is None: binary = [ None for i in range(num_nodes) ]
- return [ start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]) for i in range(num_nodes) ]
+ if extra_args is None: extra_args = [ None for _ in range(num_nodes) ]
+ if binary is None: binary = [ None for _ in range(num_nodes) ]
+ rpcs = []
+ try:
+ for i in range(num_nodes):
+ rpcs.append(start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]))
+ except: # If one node failed to start, stop the others
+ stop_nodes(rpcs)
+ raise
+ return rpcs
def log_filename(dirname, n_node, logname):
return os.path.join(dirname, "node"+str(n_node), "regtest", logname)
def stop_node(node, i):
- node.stop()
- bitcoind_processes[i].wait()
+ try:
+ node.stop()
+ except http.client.CannotSendRequest as e:
+ print("WARN: Unable to stop node: " + repr(e))
+ bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
del bitcoind_processes[i]
def stop_nodes(nodes):
for node in nodes:
- node.stop()
+ try:
+ node.stop()
+ except http.client.CannotSendRequest as e:
+ print("WARN: Unable to stop node: " + repr(e))
del nodes[:] # Emptying array closes connections as a side effect
+ wait_bitcoinds()
def set_node_times(nodes, t):
for node in nodes:
@@ -267,7 +373,7 @@ def set_node_times(nodes, t):
def wait_bitcoinds():
# Wait for all bitcoinds to cleanly exit
for bitcoind in bitcoind_processes.values():
- bitcoind.wait()
+ bitcoind.wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
bitcoind_processes.clear()
def connect_nodes(from_connection, node_num):
@@ -388,6 +494,15 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
return (txid, signresult["hex"], fee)
+def assert_fee_amount(fee, tx_size, fee_per_kB):
+ """Assert the fee was in range"""
+ target_fee = tx_size * fee_per_kB / 1000
+ if fee < target_fee:
+ raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)"%(str(fee), str(target_fee)))
+ # allow the wallet's estimation to be at most 2 bytes off
+ if fee > (tx_size + 2) * fee_per_kB / 1000:
+ raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)"%(str(fee), str(target_fee)))
+
def assert_equal(thing1, thing2):
if thing1 != thing2:
raise AssertionError("%s != %s"%(str(thing1),str(thing2)))
@@ -406,5 +521,133 @@ def assert_raises(exc, fun, *args, **kwds):
else:
raise AssertionError("No exception raised")
+def assert_is_hex_string(string):
+ try:
+ int(string, 16)
+ except Exception as e:
+ raise AssertionError(
+ "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e))
+
+def assert_is_hash_string(string, length=64):
+ if not isinstance(string, str):
+ raise AssertionError("Expected a string, got type %r" % type(string))
+ elif length and len(string) != length:
+ raise AssertionError(
+ "String of length %d expected; got %d" % (length, len(string)))
+ elif not re.match('[abcdef0-9]+$', string):
+ raise AssertionError(
+ "String %r contains invalid characters for a hash." % string)
+
+def assert_array_result(object_array, to_match, expected, should_not_find = False):
+ """
+ Pass in array of JSON objects, a dictionary with key/value pairs
+ to match against, and another dictionary with expected key/value
+ pairs.
+ If the should_not_find flag is true, to_match should not be found
+ in object_array
+ """
+ if should_not_find == True:
+ assert_equal(expected, { })
+ num_matched = 0
+ for item in object_array:
+ all_match = True
+ for key,value in to_match.items():
+ if item[key] != value:
+ all_match = False
+ if not all_match:
+ continue
+ elif should_not_find == True:
+ num_matched = num_matched+1
+ for key,value in expected.items():
+ if item[key] != value:
+ raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value)))
+ num_matched = num_matched+1
+ if num_matched == 0 and should_not_find != True:
+ raise AssertionError("No objects matched %s"%(str(to_match)))
+ if num_matched > 0 and should_not_find == True:
+ raise AssertionError("Objects were found %s"%(str(to_match)))
+
def satoshi_round(amount):
- return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
+ return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
+
+# Helper to create at least "count" utxos
+# Pass in a fee that is sufficient for relay and mining new transactions.
+def create_confirmed_utxos(fee, node, count):
+ node.generate(int(0.5*count)+101)
+ utxos = node.listunspent()
+ iterations = count - len(utxos)
+ addr1 = node.getnewaddress()
+ addr2 = node.getnewaddress()
+ if iterations <= 0:
+ return utxos
+ for i in range(iterations):
+ t = utxos.pop()
+ inputs = []
+ inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
+ outputs = {}
+ send_value = t['amount'] - fee
+ outputs[addr1] = satoshi_round(send_value/2)
+ outputs[addr2] = satoshi_round(send_value/2)
+ raw_tx = node.createrawtransaction(inputs, outputs)
+ signed_tx = node.signrawtransaction(raw_tx)["hex"]
+ txid = node.sendrawtransaction(signed_tx)
+
+ while (node.getmempoolinfo()['size'] > 0):
+ node.generate(1)
+
+ utxos = node.listunspent()
+ assert(len(utxos) >= count)
+ return utxos
+
+# Create large OP_RETURN txouts that can be appended to a transaction
+# to make it large (helper for constructing large transactions).
+def gen_return_txouts():
+ # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create
+ # So we have big transactions (and therefore can't fit very many into each block)
+ # create one script_pubkey
+ script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes
+ for i in range (512):
+ script_pubkey = script_pubkey + "01"
+ # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change
+ txouts = "81"
+ for k in range(128):
+ # add txout value
+ txouts = txouts + "0000000000000000"
+ # add length of script_pubkey
+ txouts = txouts + "fd0402"
+ # add script_pubkey
+ txouts = txouts + script_pubkey
+ return txouts
+
+def create_tx(node, coinbase, to_address, amount):
+ inputs = [{ "txid" : coinbase, "vout" : 0}]
+ outputs = { to_address : amount }
+ rawtx = node.createrawtransaction(inputs, outputs)
+ signresult = node.signrawtransaction(rawtx)
+ assert_equal(signresult["complete"], True)
+ return signresult["hex"]
+
+# Create a spend of each passed-in utxo, splicing in "txouts" to each raw
+# transaction to make it large. See gen_return_txouts() above.
+def create_lots_of_big_transactions(node, txouts, utxos, fee):
+ addr = node.getnewaddress()
+ txids = []
+ for i in range(len(utxos)):
+ t = utxos.pop()
+ inputs = []
+ inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
+ outputs = {}
+ send_value = t['amount'] - fee
+ outputs[addr] = satoshi_round(send_value)
+ rawtx = node.createrawtransaction(inputs, outputs)
+ newtx = rawtx[0:92]
+ newtx = newtx + txouts
+ newtx = newtx + rawtx[94:]
+ signresult = node.signrawtransaction(newtx, None, None, "NONE")
+ txid = node.sendrawtransaction(signresult["hex"], True)
+ txids.append(txid)
+ return txids
+
+def get_bip9_status(node, key):
+ info = node.getblockchaininfo()
+ return info['bip9_softforks'][key]
generated by cgit v1.2.3 (git 2.26.2) at 2024-10-12 18:26:47 +0000