aboutsummaryrefslogtreecommitdiff
path: root/qa/rpc-tests/test_framework
diff options
context:
space:
mode:
Diffstat (limited to 'qa/rpc-tests/test_framework')
-rw-r--r--qa/rpc-tests/test_framework/address.py74
-rw-r--r--qa/rpc-tests/test_framework/authproxy.py30
-rw-r--r--qa/rpc-tests/test_framework/blockstore.py9
-rwxr-xr-xqa/rpc-tests/test_framework/comptool.py19
-rw-r--r--qa/rpc-tests/test_framework/coverage.py4
-rw-r--r--qa/rpc-tests/test_framework/key.py25
-rwxr-xr-xqa/rpc-tests/test_framework/mininode.py438
-rw-r--r--qa/rpc-tests/test_framework/netutil.py2
-rw-r--r--qa/rpc-tests/test_framework/script.py2
-rw-r--r--qa/rpc-tests/test_framework/siphash.py64
-rwxr-xr-xqa/rpc-tests/test_framework/test_framework.py31
-rw-r--r--qa/rpc-tests/test_framework/util.py139
12 files changed, 681 insertions, 156 deletions
diff --git a/qa/rpc-tests/test_framework/address.py b/qa/rpc-tests/test_framework/address.py
new file mode 100644
index 0000000000..50b999be61
--- /dev/null
+++ b/qa/rpc-tests/test_framework/address.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# address.py
+#
+# This file encodes and decodes BASE58 P2PKH and P2SH addresses
+#
+
+from .script import hash256, hash160, sha256, CScript, OP_0
+from .util import bytes_to_hex_str, hex_str_to_bytes
+
+chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+def byte_to_base58(b, version):
+ result = ''
+ str = bytes_to_hex_str(b)
+ str = bytes_to_hex_str(chr(version).encode('latin-1')) + str
+ checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str)))
+ str += checksum[:8]
+ value = int('0x'+str,0)
+ while value > 0:
+ result = chars[value % 58] + result
+ value //= 58
+ while (str[:2] == '00'):
+ result = chars[0] + result
+ str = str[2:]
+ return result
+
+# TODO: def base58_decode
+
+def keyhash_to_p2pkh(hash, main = False):
+ assert (len(hash) == 20)
+ version = 0 if main else 111
+ return byte_to_base58(hash, version)
+
+def scripthash_to_p2sh(hash, main = False):
+ assert (len(hash) == 20)
+ version = 5 if main else 196
+ return byte_to_base58(hash, version)
+
+def key_to_p2pkh(key, main = False):
+ key = check_key(key)
+ return keyhash_to_p2pkh(hash160(key), main)
+
+def script_to_p2sh(script, main = False):
+ script = check_script(script)
+ return scripthash_to_p2sh(hash160(script), main)
+
+def key_to_p2sh_p2wpkh(key, main = False):
+ key = check_key(key)
+ p2shscript = CScript([OP_0, hash160(key)])
+ return script_to_p2sh(p2shscript, main)
+
+def script_to_p2sh_p2wsh(script, main = False):
+ script = check_script(script)
+ p2shscript = CScript([OP_0, sha256(script)])
+ return script_to_p2sh(p2shscript, main)
+
+def check_key(key):
+ if (type(key) is str):
+ key = hex_str_to_bytes(key) # Assuming this is hex string
+ if (type(key) is bytes and (len(key) == 33 or len(key) == 65)):
+ return key
+ assert(False)
+
+def check_script(script):
+ if (type(script) is str):
+ script = hex_str_to_bytes(script) # Assuming this is hex string
+ if (type(script) is bytes or type(script) is CScript):
+ return script
+ assert(False)
diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py
index d095a56ce7..09ed611299 100644
--- a/qa/rpc-tests/test_framework/authproxy.py
+++ b/qa/rpc-tests/test_framework/authproxy.py
@@ -1,6 +1,6 @@
"""
- Copyright 2011 Jeff Garzik
+ Copyright (c) 2011 Jeff Garzik
AuthServiceProxy has the following improvements over python-jsonrpc's
ServiceProxy class:
@@ -42,6 +42,7 @@ import base64
import decimal
import json
import logging
+import socket
try:
import urllib.parse as urlparse
except ImportError:
@@ -55,7 +56,11 @@ log = logging.getLogger("BitcoinRPC")
class JSONRPCException(Exception):
def __init__(self, rpc_error):
- Exception.__init__(self)
+ try:
+ errmsg = '%(message)s (%(code)i)' % rpc_error
+ except (KeyError, TypeError):
+ errmsg = ''
+ Exception.__init__(self, errmsg)
self.error = rpc_error
@@ -126,20 +131,23 @@ class AuthServiceProxy(object):
return self._get_response()
else:
raise
- except BrokenPipeError:
- # Python 3.5+ raises this instead of BadStatusLine when the connection was reset
+ except (BrokenPipeError,ConnectionResetError):
+ # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset
+ # ConnectionResetError happens on FreeBSD with Python 3.4
self.__conn.close()
self.__conn.request(method, path, postdata, headers)
return self._get_response()
- def __call__(self, *args):
+ def __call__(self, *args, **argsn):
AuthServiceProxy.__id_count += 1
log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name,
json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
+ if args and argsn:
+ raise ValueError('Cannot handle both named and positional arguments')
postdata = json.dumps({'version': '1.1',
'method': self._service_name,
- 'params': args,
+ 'params': args or argsn,
'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
response = self._request('POST', self.__url.path, postdata.encode('utf-8'))
if response['error'] is not None:
@@ -156,7 +164,15 @@ class AuthServiceProxy(object):
return self._request('POST', self.__url.path, postdata.encode('utf-8'))
def _get_response(self):
- http_response = self.__conn.getresponse()
+ try:
+ http_response = self.__conn.getresponse()
+ except socket.timeout as e:
+ raise JSONRPCException({
+ 'code': -344,
+ 'message': '%r RPC took longer than %f seconds. Consider '
+ 'using larger timeout for calls that take '
+ 'longer to return.' % (self._service_name,
+ self.__conn.timeout)})
if http_response is None:
raise JSONRPCException({
'code': -342, 'message': 'missing HTTP response from server'})
diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py
index 6120dd574b..28a6b92b81 100644
--- a/qa/rpc-tests/test_framework/blockstore.py
+++ b/qa/rpc-tests/test_framework/blockstore.py
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
+# Copyright (c) 2015-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# BlockStore: a helper class that keeps a map of blocks and implements
# helper functions for responding to getheaders and getdata,
# and for constructing a getheaders message
@@ -6,11 +9,11 @@
from .mininode import *
from io import BytesIO
-import dbm.ndbm
+import dbm.dumb as dbmd
class BlockStore(object):
def __init__(self, datadir):
- self.blockDB = dbm.ndbm.open(datadir + "/blocks", 'c')
+ self.blockDB = dbmd.open(datadir + "/blocks", 'c')
self.currentBlock = 0
self.headers_map = dict()
@@ -120,7 +123,7 @@ class BlockStore(object):
class TxStore(object):
def __init__(self, datadir):
- self.txDB = dbm.ndbm.open(datadir + "/transactions", 'c')
+ self.txDB = dbmd.open(datadir + "/transactions", 'c')
def close(self):
self.txDB.close()
diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py
index 7c92d3f828..17679fc7e1 100755
--- a/qa/rpc-tests/test_framework/comptool.py
+++ b/qa/rpc-tests/test_framework/comptool.py
@@ -111,6 +111,11 @@ class TestNode(NodeConnCB):
m.locator = self.block_store.get_locator(self.bestblockhash)
self.conn.send_message(m)
+ def send_header(self, header):
+ m = msg_headers()
+ m.headers.append(header)
+ self.conn.send_message(m)
+
# This assumes BIP31
def send_ping(self, nonce):
self.pingMap[nonce] = True
@@ -345,8 +350,16 @@ class TestManager(object):
# Either send inv's to each node and sync, or add
# to invqueue for later inv'ing.
if (test_instance.sync_every_block):
- [ c.cb.send_inv(block) for c in self.connections ]
- self.sync_blocks(block.sha256, 1)
+ # if we expect success, send inv and sync every block
+ # if we expect failure, just push the block and see what happens.
+ if outcome == True:
+ [ c.cb.send_inv(block) for c in self.connections ]
+ self.sync_blocks(block.sha256, 1)
+ else:
+ [ c.send_message(msg_block(block)) for c in self.connections ]
+ [ c.cb.send_ping(self.ping_counter) for c in self.connections ]
+ self.wait_for_pings(self.ping_counter)
+ self.ping_counter += 1
if (not self.check_results(tip, outcome)):
raise AssertionError("Test failed at test %d" % test_number)
else:
@@ -354,6 +367,8 @@ class TestManager(object):
elif isinstance(b_or_t, CBlockHeader):
block_header = b_or_t
self.block_store.add_header(block_header)
+ [ c.cb.send_header(block_header) for c in self.connections ]
+
else: # Tx test runner
assert(isinstance(b_or_t, CTransaction))
tx = b_or_t
diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py
index 23fce61014..13b33869f5 100644
--- a/qa/rpc-tests/test_framework/coverage.py
+++ b/qa/rpc-tests/test_framework/coverage.py
@@ -50,7 +50,7 @@ class AuthServiceProxyWrapper(object):
rpc_method = self.auth_service_proxy_instance._service_name
if self.coverage_logfile:
- with open(self.coverage_logfile, 'a+') as f:
+ with open(self.coverage_logfile, 'a+', encoding='utf8') as f:
f.write("%s\n" % rpc_method)
return return_val
@@ -100,7 +100,7 @@ def write_all_rpc_commands(dirname, node):
if line and not line.startswith('='):
commands.add("%s\n" % line.split()[0])
- with open(filename, 'w') as f:
+ with open(filename, 'w', encoding='utf8') as f:
f.writelines(list(commands))
return True
diff --git a/qa/rpc-tests/test_framework/key.py b/qa/rpc-tests/test_framework/key.py
index ba3038fe04..c63a15c1e0 100644
--- a/qa/rpc-tests/test_framework/key.py
+++ b/qa/rpc-tests/test_framework/key.py
@@ -75,6 +75,9 @@ ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p,
# this specifies the curve used with ECDSA.
NID_secp256k1 = 714 # from openssl/obj_mac.h
+SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141
+SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2
+
# Thx to Sam Devlin for the ctypes magic 64-bit fix.
def _check_result(val, func, args):
if val == 0:
@@ -147,7 +150,7 @@ class CECKey(object):
r = self.get_raw_ecdh_key(other_pubkey)
return kdf(r)
- def sign(self, hash):
+ def sign(self, hash, low_s = True):
# FIXME: need unit tests for below cases
if not isinstance(hash, bytes):
raise TypeError('Hash must be bytes instance; got %r' % hash.__class__)
@@ -159,7 +162,25 @@ class CECKey(object):
mb_sig = ctypes.create_string_buffer(sig_size0.value)
result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k)
assert 1 == result
- return mb_sig.raw[:sig_size0.value]
+ assert mb_sig.raw[0] == 0x30
+ assert mb_sig.raw[1] == sig_size0.value - 2
+ total_size = mb_sig.raw[1]
+ assert mb_sig.raw[2] == 2
+ r_size = mb_sig.raw[3]
+ assert mb_sig.raw[4 + r_size] == 2
+ s_size = mb_sig.raw[5 + r_size]
+ s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big')
+ if (not low_s) or s_value <= SECP256K1_ORDER_HALF:
+ return mb_sig.raw[:sig_size0.value]
+ else:
+ low_s_value = SECP256K1_ORDER - s_value
+ low_s_bytes = (low_s_value).to_bytes(33, byteorder='big')
+ while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80:
+ low_s_bytes = low_s_bytes[1:]
+ new_s_size = len(low_s_bytes)
+ new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big')
+ new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big')
+ return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes
def verify(self, hash, sig):
"""Verify a DER signature"""
diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index cdd5292cd6..91daa4ab1f 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -36,13 +36,15 @@ from threading import RLock
from threading import Thread
import logging
import copy
+from test_framework.siphash import siphash256
BIP0031_VERSION = 60000
-MY_VERSION = 60001 # past bip-31 for ping/pong
+MY_VERSION = 70014 # past bip-31 for ping/pong
MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
+MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
MAX_INV_SZ = 50000
-MAX_BLOCK_SIZE = 1000000
+MAX_BLOCK_BASE_SIZE = 1000000
COIN = 100000000 # 1 btc in satoshis
@@ -52,7 +54,7 @@ NODE_BLOOM = (1 << 2)
NODE_WITNESS = (1 << 3)
# Keep our own socket map for asyncore, so that we can track disconnects
-# ourselves (to workaround an issue with closing an asyncore socket when
+# ourselves (to workaround an issue with closing an asyncore socket when
# using select)
mininode_socket_map = dict()
@@ -74,8 +76,19 @@ def ripemd160(s):
def hash256(s):
return sha256(sha256(s))
+def ser_compact_size(l):
+ r = b""
+ if l < 253:
+ r = struct.pack("B", l)
+ elif l < 0x10000:
+ r = struct.pack("<BH", 253, l)
+ elif l < 0x100000000:
+ r = struct.pack("<BI", 254, l)
+ else:
+ r = struct.pack("<BQ", 255, l)
+ return r
-def deser_string(f):
+def deser_compact_size(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
@@ -83,16 +96,14 @@ def deser_string(f):
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
+ return nit
+
+def deser_string(f):
+ nit = deser_compact_size(f)
return f.read(nit)
def ser_string(s):
- if len(s) < 253:
- return struct.pack("B", len(s)) + s
- elif len(s) < 0x10000:
- return struct.pack("<BH", 253, len(s)) + s
- elif len(s) < 0x100000000:
- return struct.pack("<BI", 254, len(s)) + s
- return struct.pack("<BQ", 255, len(s)) + s
+ return ser_compact_size(len(s)) + s
def deser_uint256(f):
r = 0
@@ -125,13 +136,7 @@ def uint256_from_compact(c):
def deser_vector(f, c):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
for i in range(nit):
t = c()
@@ -144,15 +149,7 @@ def deser_vector(f, c):
# entries in the vector (we use this for serializing the vector of transactions
# for a witness block).
def ser_vector(l, ser_function_name=None):
- r = b""
- if len(l) < 253:
- r = struct.pack("B", len(l))
- elif len(l) < 0x10000:
- r = struct.pack("<BH", 253, len(l))
- elif len(l) < 0x100000000:
- r = struct.pack("<BI", 254, len(l))
- else:
- r = struct.pack("<BQ", 255, len(l))
+ r = ser_compact_size(len(l))
for i in l:
if ser_function_name:
r += getattr(i, ser_function_name)()
@@ -162,13 +159,7 @@ def ser_vector(l, ser_function_name=None):
def deser_uint256_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
for i in range(nit):
t = deser_uint256(f)
@@ -177,28 +168,14 @@ def deser_uint256_vector(f):
def ser_uint256_vector(l):
- r = b""
- if len(l) < 253:
- r = struct.pack("B", len(l))
- elif len(l) < 0x10000:
- r = struct.pack("<BH", 253, len(l))
- elif len(l) < 0x100000000:
- r = struct.pack("<BI", 254, len(l))
- else:
- r = struct.pack("<BQ", 255, len(l))
+ r = ser_compact_size(len(l))
for i in l:
r += ser_uint256(i)
return r
def deser_string_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
for i in range(nit):
t = deser_string(f)
@@ -207,28 +184,14 @@ def deser_string_vector(f):
def ser_string_vector(l):
- r = b""
- if len(l) < 253:
- r = struct.pack("B", len(l))
- elif len(l) < 0x10000:
- r = struct.pack("<BH", 253, len(l))
- elif len(l) < 0x100000000:
- r = struct.pack("<BI", 254, len(l))
- else:
- r = struct.pack("<BQ", 255, len(l))
+ r = ser_compact_size(len(l))
for sv in l:
r += ser_string(sv)
return r
def deser_int_vector(f):
- nit = struct.unpack("<B", f.read(1))[0]
- if nit == 253:
- nit = struct.unpack("<H", f.read(2))[0]
- elif nit == 254:
- nit = struct.unpack("<I", f.read(4))[0]
- elif nit == 255:
- nit = struct.unpack("<Q", f.read(8))[0]
+ nit = deser_compact_size(f)
r = []
for i in range(nit):
t = struct.unpack("<i", f.read(4))[0]
@@ -237,15 +200,7 @@ def deser_int_vector(f):
def ser_int_vector(l):
- r = b""
- if len(l) < 253:
- r = struct.pack("B", len(l))
- elif len(l) < 0x10000:
- r = struct.pack("<BH", 253, len(l))
- elif len(l) < 0x100000000:
- r = struct.pack("<BI", 254, len(l))
- else:
- r = struct.pack("<BQ", 255, len(l))
+ r = ser_compact_size(len(l))
for i in l:
r += struct.pack("<i", i)
return r
@@ -294,7 +249,8 @@ class CInv(object):
1: "TX",
2: "Block",
1|MSG_WITNESS_FLAG: "WitnessTx",
- 2|MSG_WITNESS_FLAG : "WitnessBlock"
+ 2|MSG_WITNESS_FLAG : "WitnessBlock",
+ 4: "CompactBlock"
}
def __init__(self, t=0, h=0):
@@ -497,7 +453,7 @@ class CTransaction(object):
else:
self.vout = deser_vector(f, CTxOut)
if flags != 0:
- self.wit.vtxinwit = [CTxInWitness()]*len(self.vin)
+ self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
self.wit.deserialize(f)
self.nLockTime = struct.unpack("<I", f.read(4))[0]
self.sha256 = None
@@ -563,8 +519,8 @@ class CTransaction(object):
return True
def __repr__(self):
- return "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i)" \
- % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime)
+ return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
+ % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
class CBlockHeader(object):
@@ -781,6 +737,208 @@ class CAlert(object):
% (len(self.vchMsg), len(self.vchSig))
+class PrefilledTransaction(object):
+ def __init__(self, index=0, tx = None):
+ self.index = index
+ self.tx = tx
+
+ def deserialize(self, f):
+ self.index = deser_compact_size(f)
+ self.tx = CTransaction()
+ self.tx.deserialize(f)
+
+ def serialize(self, with_witness=False):
+ r = b""
+ r += ser_compact_size(self.index)
+ if with_witness:
+ r += self.tx.serialize_with_witness()
+ else:
+ r += self.tx.serialize_without_witness()
+ return r
+
+ def serialize_with_witness(self):
+ return self.serialize(with_witness=True)
+
+ def __repr__(self):
+ return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
+
+# This is what we send on the wire, in a cmpctblock message.
+class P2PHeaderAndShortIDs(object):
+ def __init__(self):
+ self.header = CBlockHeader()
+ self.nonce = 0
+ self.shortids_length = 0
+ self.shortids = []
+ self.prefilled_txn_length = 0
+ self.prefilled_txn = []
+
+ def deserialize(self, f):
+ self.header.deserialize(f)
+ self.nonce = struct.unpack("<Q", f.read(8))[0]
+ self.shortids_length = deser_compact_size(f)
+ for i in range(self.shortids_length):
+ # shortids are defined to be 6 bytes in the spec, so append
+ # two zero bytes and read it in as an 8-byte number
+ self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
+ self.prefilled_txn = deser_vector(f, PrefilledTransaction)
+ self.prefilled_txn_length = len(self.prefilled_txn)
+
+ # When using version 2 compact blocks, we must serialize with_witness.
+ def serialize(self, with_witness=False):
+ r = b""
+ r += self.header.serialize()
+ r += struct.pack("<Q", self.nonce)
+ r += ser_compact_size(self.shortids_length)
+ for x in self.shortids:
+ # We only want the first 6 bytes
+ r += struct.pack("<Q", x)[0:6]
+ if with_witness:
+ r += ser_vector(self.prefilled_txn, "serialize_with_witness")
+ else:
+ r += ser_vector(self.prefilled_txn)
+ return r
+
+ def __repr__(self):
+ return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
+
+# P2P version of the above that will use witness serialization (for compact
+# block version 2)
+class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
+ def serialize(self):
+ return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
+
+# Calculate the BIP 152-compact blocks shortid for a given transaction hash
+def calculate_shortid(k0, k1, tx_hash):
+ expected_shortid = siphash256(k0, k1, tx_hash)
+ expected_shortid &= 0x0000ffffffffffff
+ return expected_shortid
+
+# This version gets rid of the array lengths, and reinterprets the differential
+# encoding into indices that can be used for lookup.
+class HeaderAndShortIDs(object):
+ def __init__(self, p2pheaders_and_shortids = None):
+ self.header = CBlockHeader()
+ self.nonce = 0
+ self.shortids = []
+ self.prefilled_txn = []
+ self.use_witness = False
+
+ if p2pheaders_and_shortids != None:
+ self.header = p2pheaders_and_shortids.header
+ self.nonce = p2pheaders_and_shortids.nonce
+ self.shortids = p2pheaders_and_shortids.shortids
+ last_index = -1
+ for x in p2pheaders_and_shortids.prefilled_txn:
+ self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
+ last_index = self.prefilled_txn[-1].index
+
+ def to_p2p(self):
+ if self.use_witness:
+ ret = P2PHeaderAndShortWitnessIDs()
+ else:
+ ret = P2PHeaderAndShortIDs()
+ ret.header = self.header
+ ret.nonce = self.nonce
+ ret.shortids_length = len(self.shortids)
+ ret.shortids = self.shortids
+ ret.prefilled_txn_length = len(self.prefilled_txn)
+ ret.prefilled_txn = []
+ last_index = -1
+ for x in self.prefilled_txn:
+ ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
+ last_index = x.index
+ return ret
+
+ def get_siphash_keys(self):
+ header_nonce = self.header.serialize()
+ header_nonce += struct.pack("<Q", self.nonce)
+ hash_header_nonce_as_str = sha256(header_nonce)
+ key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
+ key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
+ return [ key0, key1 ]
+
+ # Version 2 compact blocks use wtxid in shortids (rather than txid)
+ def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
+ self.header = CBlockHeader(block)
+ self.nonce = nonce
+ self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
+ self.shortids = []
+ self.use_witness = use_witness
+ [k0, k1] = self.get_siphash_keys()
+ for i in range(len(block.vtx)):
+ if i not in prefill_list:
+ tx_hash = block.vtx[i].sha256
+ if use_witness:
+ tx_hash = block.vtx[i].calc_sha256(with_witness=True)
+ self.shortids.append(calculate_shortid(k0, k1, tx_hash))
+
+ def __repr__(self):
+ return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
+
+
+class BlockTransactionsRequest(object):
+
+ def __init__(self, blockhash=0, indexes = None):
+ self.blockhash = blockhash
+ self.indexes = indexes if indexes != None else []
+
+ def deserialize(self, f):
+ self.blockhash = deser_uint256(f)
+ indexes_length = deser_compact_size(f)
+ for i in range(indexes_length):
+ self.indexes.append(deser_compact_size(f))
+
+ def serialize(self):
+ r = b""
+ r += ser_uint256(self.blockhash)
+ r += ser_compact_size(len(self.indexes))
+ for x in self.indexes:
+ r += ser_compact_size(x)
+ return r
+
+ # helper to set the differentially encoded indexes from absolute ones
+ def from_absolute(self, absolute_indexes):
+ self.indexes = []
+ last_index = -1
+ for x in absolute_indexes:
+ self.indexes.append(x-last_index-1)
+ last_index = x
+
+ def to_absolute(self):
+ absolute_indexes = []
+ last_index = -1
+ for x in self.indexes:
+ absolute_indexes.append(x+last_index+1)
+ last_index = absolute_indexes[-1]
+ return absolute_indexes
+
+ def __repr__(self):
+ return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
+
+
+class BlockTransactions(object):
+
+ def __init__(self, blockhash=0, transactions = None):
+ self.blockhash = blockhash
+ self.transactions = transactions if transactions != None else []
+
+ def deserialize(self, f):
+ self.blockhash = deser_uint256(f)
+ self.transactions = deser_vector(f, CTransaction)
+
+ def serialize(self, with_witness=False):
+ r = b""
+ r += ser_uint256(self.blockhash)
+ if with_witness:
+ r += ser_vector(self.transactions, "serialize_with_witness")
+ else:
+ r += ser_vector(self.transactions)
+ return r
+
+ def __repr__(self):
+ return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
+
+
# Objects that correspond to messages on the wire
class msg_version(object):
command = b"version"
@@ -794,6 +952,7 @@ class msg_version(object):
self.nNonce = random.getrandbits(64)
self.strSubVer = MY_SUBVERSION
self.nStartingHeight = -1
+ self.nRelay = MY_RELAY
def deserialize(self, f):
self.nVersion = struct.unpack("<i", f.read(4))[0]
@@ -803,21 +962,32 @@ class msg_version(object):
self.nTime = struct.unpack("<q", f.read(8))[0]
self.addrTo = CAddress()
self.addrTo.deserialize(f)
+
if self.nVersion >= 106:
self.addrFrom = CAddress()
self.addrFrom.deserialize(f)
self.nNonce = struct.unpack("<Q", f.read(8))[0]
self.strSubVer = deser_string(f)
- if self.nVersion >= 209:
- self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
- else:
- self.nStartingHeight = None
else:
self.addrFrom = None
self.nNonce = None
self.strSubVer = None
self.nStartingHeight = None
+ if self.nVersion >= 209:
+ self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
+ else:
+ self.nStartingHeight = None
+
+ if self.nVersion >= 70001:
+ # Relay field is optional for version 70001 onwards
+ try:
+ self.nRelay = struct.unpack("<b", f.read(1))[0]
+ except:
+ self.nRelay = 0
+ else:
+ self.nRelay = 0
+
def serialize(self):
r = b""
r += struct.pack("<i", self.nVersion)
@@ -828,13 +998,14 @@ class msg_version(object):
r += struct.pack("<Q", self.nNonce)
r += ser_string(self.strSubVer)
r += struct.pack("<i", self.nStartingHeight)
+ r += struct.pack("<b", self.nRelay)
return r
def __repr__(self):
- return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i)' \
+ return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
% (self.nVersion, self.nServices, time.ctime(self.nTime),
repr(self.addrTo), repr(self.addrFrom), self.nNonce,
- self.strSubVer, self.nStartingHeight)
+ self.strSubVer, self.nStartingHeight, self.nRelay)
class msg_verack(object):
@@ -1184,7 +1355,7 @@ class msg_reject(object):
% (self.message, self.code, self.reason, self.data)
# Helper function
-def wait_until(predicate, attempts=float('inf'), timeout=float('inf')):
+def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')):
attempt = 0
elapsed = 0
@@ -1215,6 +1386,85 @@ class msg_feefilter(object):
def __repr__(self):
return "msg_feefilter(feerate=%08x)" % self.feerate
+class msg_sendcmpct(object):
+ command = b"sendcmpct"
+
+ def __init__(self):
+ self.announce = False
+ self.version = 1
+
+ def deserialize(self, f):
+ self.announce = struct.unpack("<?", f.read(1))[0]
+ self.version = struct.unpack("<Q", f.read(8))[0]
+
+ def serialize(self):
+ r = b""
+ r += struct.pack("<?", self.announce)
+ r += struct.pack("<Q", self.version)
+ return r
+
+ def __repr__(self):
+ return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
+
+class msg_cmpctblock(object):
+ command = b"cmpctblock"
+
+ def __init__(self, header_and_shortids = None):
+ self.header_and_shortids = header_and_shortids
+
+ def deserialize(self, f):
+ self.header_and_shortids = P2PHeaderAndShortIDs()
+ self.header_and_shortids.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.header_and_shortids.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
+
+class msg_getblocktxn(object):
+ command = b"getblocktxn"
+
+ def __init__(self):
+ self.block_txn_request = None
+
+ def deserialize(self, f):
+ self.block_txn_request = BlockTransactionsRequest()
+ self.block_txn_request.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.block_txn_request.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
+
+class msg_blocktxn(object):
+ command = b"blocktxn"
+
+ def __init__(self):
+ self.block_transactions = BlockTransactions()
+
+ def deserialize(self, f):
+ self.block_transactions.deserialize(f)
+
+ def serialize(self):
+ r = b""
+ r += self.block_transactions.serialize()
+ return r
+
+ def __repr__(self):
+ return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
+
+class msg_witness_blocktxn(msg_blocktxn):
+ def serialize(self):
+ r = b""
+ r += self.block_transactions.serialize(with_witness=True)
+ return r
+
# This is what a callback should look like for NodeConn
# Reimplement the on_* functions to provide handling for events
class NodeConnCB(object):
@@ -1295,6 +1545,10 @@ class NodeConnCB(object):
def on_pong(self, conn, message): pass
def on_feefilter(self, conn, message): pass
def on_sendheaders(self, conn, message): pass
+ def on_sendcmpct(self, conn, message): pass
+ def on_cmpctblock(self, conn, message): pass
+ def on_getblocktxn(self, conn, message): pass
+ def on_blocktxn(self, conn, message): pass
# More useful callbacks and functions for NodeConnCB's which have a single NodeConn
class SingleNodeConnCB(NodeConnCB):
@@ -1311,6 +1565,10 @@ class SingleNodeConnCB(NodeConnCB):
def send_message(self, message):
self.connection.send_message(message)
+ def send_and_ping(self, message):
+ self.send_message(message)
+ self.sync_with_ping()
+
def on_pong(self, conn, message):
self.last_pong = message
@@ -1319,7 +1577,7 @@ class SingleNodeConnCB(NodeConnCB):
def received_pong():
return (self.last_pong.nonce == self.ping_counter)
self.send_message(msg_ping(nonce=self.ping_counter))
- success = wait_until(received_pong, timeout)
+ success = wait_until(received_pong, timeout=timeout)
self.ping_counter += 1
return success
@@ -1344,7 +1602,11 @@ class NodeConn(asyncore.dispatcher):
b"reject": msg_reject,
b"mempool": msg_mempool,
b"feefilter": msg_feefilter,
- b"sendheaders": msg_sendheaders
+ b"sendheaders": msg_sendheaders,
+ b"sendcmpct": msg_sendcmpct,
+ b"cmpctblock": msg_cmpctblock,
+ b"getblocktxn": msg_getblocktxn,
+ b"blocktxn": msg_blocktxn
}
MAGIC_BYTES = {
"mainnet": b"\xf9\xbe\xb4\xd9", # mainnet
diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py
index 573b06772d..b92a9f6e1c 100644
--- a/qa/rpc-tests/test_framework/netutil.py
+++ b/qa/rpc-tests/test_framework/netutil.py
@@ -58,7 +58,7 @@ def netstat(typ='tcp'):
To get pid of all network process running on system, you must run this script
as superuser
'''
- with open('/proc/net/'+typ,'r') as f:
+ with open('/proc/net/'+typ,'r',encoding='utf8') as f:
content = f.readlines()
content.pop(0)
result = []
diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py
index b46c643ccb..83bbf20479 100644
--- a/qa/rpc-tests/test_framework/script.py
+++ b/qa/rpc-tests/test_framework/script.py
@@ -882,7 +882,7 @@ def SignatureHash(script, txTo, inIdx, hashtype):
tmp = txtmp.vout[outIdx]
txtmp.vout = []
for i in range(outIdx):
- txtmp.vout.append(CTxOut())
+ txtmp.vout.append(CTxOut(-1))
txtmp.vout.append(tmp)
for i in range(len(txtmp.vin)):
diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py
new file mode 100644
index 0000000000..9c0574bd93
--- /dev/null
+++ b/qa/rpc-tests/test_framework/siphash.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# siphash.py - Specialized SipHash-2-4 implementations
+#
+# This implements SipHash-2-4 for 256-bit integers.
+
+def rotl64(n, b):
+ return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
+
+def siphash_round(v0, v1, v2, v3):
+ v0 = (v0 + v1) & ((1 << 64) - 1)
+ v1 = rotl64(v1, 13)
+ v1 ^= v0
+ v0 = rotl64(v0, 32)
+ v2 = (v2 + v3) & ((1 << 64) - 1)
+ v3 = rotl64(v3, 16)
+ v3 ^= v2
+ v0 = (v0 + v3) & ((1 << 64) - 1)
+ v3 = rotl64(v3, 21)
+ v3 ^= v0
+ v2 = (v2 + v1) & ((1 << 64) - 1)
+ v1 = rotl64(v1, 17)
+ v1 ^= v2
+ v2 = rotl64(v2, 32)
+ return (v0, v1, v2, v3)
+
+def siphash256(k0, k1, h):
+ n0 = h & ((1 << 64) - 1)
+ n1 = (h >> 64) & ((1 << 64) - 1)
+ n2 = (h >> 128) & ((1 << 64) - 1)
+ n3 = (h >> 192) & ((1 << 64) - 1)
+ v0 = 0x736f6d6570736575 ^ k0
+ v1 = 0x646f72616e646f6d ^ k1
+ v2 = 0x6c7967656e657261 ^ k0
+ v3 = 0x7465646279746573 ^ k1 ^ n0
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n0
+ v3 ^= n1
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n1
+ v3 ^= n2
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n2
+ v3 ^= n3
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= n3
+ v3 ^= 0x2000000000000000
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0 ^= 0x2000000000000000
+ v2 ^= 0xFF
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+ return v0 ^ v1 ^ v2 ^ v3
diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py
index 0dfece6b27..98c4f6070b 100755
--- a/qa/rpc-tests/test_framework/test_framework.py
+++ b/qa/rpc-tests/test_framework/test_framework.py
@@ -21,7 +21,6 @@ from .util import (
sync_mempools,
stop_nodes,
stop_node,
- wait_bitcoinds,
enable_coverage,
check_json_precision,
initialize_chain_clean,
@@ -48,7 +47,7 @@ class BitcoinTestFramework(object):
if self.setup_clean_chain:
initialize_chain_clean(self.options.tmpdir, self.num_nodes)
else:
- initialize_chain(self.options.tmpdir, self.num_nodes)
+ initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir)
def stop_node(self, num_node):
stop_node(self.nodes[num_node], num_node)
@@ -81,7 +80,6 @@ class BitcoinTestFramework(object):
"""
assert not self.is_network_split
stop_nodes(self.nodes)
- wait_bitcoinds()
self.setup_network(True)
def sync_all(self):
@@ -100,7 +98,6 @@ class BitcoinTestFramework(object):
"""
assert self.is_network_split
stop_nodes(self.nodes)
- wait_bitcoinds()
self.setup_network(False)
def main(self):
@@ -112,6 +109,8 @@ class BitcoinTestFramework(object):
help="Don't stop bitcoinds after the test execution")
parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"),
help="Source directory containing bitcoind/bitcoin-cli (default: %default)")
+ parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"),
+ help="Directory for caching pregenerated datadirs")
parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
help="Root directory for datadirs")
parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true",
@@ -123,7 +122,8 @@ class BitcoinTestFramework(object):
self.add_options(parser)
(self.options, self.args) = parser.parse_args()
- self.options.tmpdir += '/' + str(self.options.port_seed)
+ # backup dir variable for removal at cleanup
+ self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed)
if self.options.trace_rpc:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
@@ -139,16 +139,11 @@ class BitcoinTestFramework(object):
success = False
try:
- if not os.path.isdir(self.options.tmpdir):
- os.makedirs(self.options.tmpdir)
+ os.makedirs(self.options.tmpdir, exist_ok=False)
self.setup_chain()
-
self.setup_network()
-
self.run_test()
-
success = True
-
except JSONRPCException as e:
print("JSONRPC error: "+e.error['message'])
traceback.print_tb(sys.exc_info()[2])
@@ -167,16 +162,26 @@ class BitcoinTestFramework(object):
if not self.options.noshutdown:
print("Stopping nodes")
stop_nodes(self.nodes)
- wait_bitcoinds()
else:
print("Note: bitcoinds were not stopped and may still be running")
if not self.options.nocleanup and not self.options.noshutdown and success:
print("Cleaning up")
shutil.rmtree(self.options.tmpdir)
+ if not os.listdir(self.options.root):
+ os.rmdir(self.options.root)
else:
print("Not cleaning up dir %s" % self.options.tmpdir)
-
+ if os.getenv("PYTHON_DEBUG", ""):
+ # Dump the end of the debug logs, to aid in debugging rare
+ # travis failures.
+ import glob
+ filenames = glob.glob(self.options.tmpdir + "/node*/regtest/debug.log")
+ MAX_LINES_TO_PRINT = 1000
+ for f in filenames:
+ print("From" , f, ":")
+ from collections import deque
+ print("".join(deque(open(f), MAX_LINES_TO_PRINT)))
if success:
print("Tests successful")
sys.exit(0)
diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py
index 32fe79efc3..aca82c8b6f 100644
--- a/qa/rpc-tests/test_framework/util.py
+++ b/qa/rpc-tests/test_framework/util.py
@@ -121,19 +121,44 @@ def hex_str_to_bytes(hex_str):
def str_to_b64str(string):
return b64encode(string.encode('utf-8')).decode('ascii')
-def sync_blocks(rpc_connections, wait=1, timeout=60):
+def sync_blocks(rpc_connections, *, wait=1, timeout=60):
"""
- Wait until everybody has the same tip
+ Wait until everybody has the same tip.
+
+ sync_blocks needs to be called with an rpc_connections set that has least
+ one node already synced to the latest, stable tip, otherwise there's a
+ chance it might return before all nodes are stably synced.
+ """
+ # Use getblockcount() instead of waitforblockheight() to determine the
+ # initial max height because the two RPCs look at different internal global
+ # variables (chainActive vs latestBlock) and the former gets updated
+ # earlier.
+ maxheight = max(x.getblockcount() for x in rpc_connections)
+ start_time = cur_time = time.time()
+ while cur_time <= start_time + timeout:
+ tips = [r.waitforblockheight(maxheight, int(wait * 1000)) for r in rpc_connections]
+ if all(t["height"] == maxheight for t in tips):
+ if all(t["hash"] == tips[0]["hash"] for t in tips):
+ return
+ raise AssertionError("Block sync failed, mismatched block hashes:{}".format(
+ "".join("\n {!r}".format(tip) for tip in tips)))
+ cur_time = time.time()
+ raise AssertionError("Block sync to height {} timed out:{}".format(
+ maxheight, "".join("\n {!r}".format(tip) for tip in tips)))
+
+def sync_chain(rpc_connections, *, wait=1, timeout=60):
+ """
+ Wait until everybody has the same best block
"""
while timeout > 0:
- tips = [ x.getbestblockhash() for x in rpc_connections ]
- if tips == [ tips[0] ]*len(tips):
- return True
+ best_hash = [x.getbestblockhash() for x in rpc_connections]
+ if best_hash == [best_hash[0]]*len(best_hash):
+ return
time.sleep(wait)
timeout -= wait
- raise AssertionError("Block sync failed")
+ raise AssertionError("Chain sync failed: Best block hashes don't match")
-def sync_mempools(rpc_connections, wait=1, timeout=60):
+def sync_mempools(rpc_connections, *, wait=1, timeout=60):
"""
Wait until everybody has the same transactions in their memory
pools
@@ -145,7 +170,7 @@ def sync_mempools(rpc_connections, wait=1, timeout=60):
if set(rpc_connections[i].getrawmempool()) == pool:
num_match = num_match+1
if num_match == len(rpc_connections):
- return True
+ return
time.sleep(wait)
timeout -= wait
raise AssertionError("Mempool sync failed")
@@ -157,7 +182,7 @@ def initialize_datadir(dirname, n):
if not os.path.isdir(datadir):
os.makedirs(datadir)
rpc_u, rpc_p = rpc_auth_pair(n)
- with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f:
+ with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f:
f.write("regtest=1\n")
f.write("rpcuser=" + rpc_u + "\n")
f.write("rpcpassword=" + rpc_p + "\n")
@@ -171,7 +196,15 @@ def rpc_auth_pair(n):
def rpc_url(i, rpchost=None):
rpc_u, rpc_p = rpc_auth_pair(i)
- return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, rpchost or '127.0.0.1', rpc_port(i))
+ host = '127.0.0.1'
+ port = rpc_port(i)
+ if rpchost:
+ parts = rpchost.split(':')
+ if len(parts) == 2:
+ host, port = parts
+ else:
+ host = rpchost
+ return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port))
def wait_for_bitcoind_start(process, url, i):
'''
@@ -190,10 +223,10 @@ def wait_for_bitcoind_start(process, url, i):
raise # unknown IO error
except JSONRPCException as e: # Initialization phase
if e.error['code'] != -28: # RPC in warmup?
- raise # unkown JSON RPC exception
+ raise # unknown JSON RPC exception
time.sleep(0.25)
-def initialize_chain(test_dir, num_nodes):
+def initialize_chain(test_dir, num_nodes, cachedir):
"""
Create a cache of a 200-block-long chain (with wallet) for MAX_NODES
Afterward, create num_nodes copies from the cache
@@ -202,7 +235,7 @@ def initialize_chain(test_dir, num_nodes):
assert num_nodes <= MAX_NODES
create_cache = False
for i in range(MAX_NODES):
- if not os.path.isdir(os.path.join('cache', 'node'+str(i))):
+ if not os.path.isdir(os.path.join(cachedir, 'node'+str(i))):
create_cache = True
break
@@ -210,12 +243,12 @@ def initialize_chain(test_dir, num_nodes):
#find and delete old cache directories if any exist
for i in range(MAX_NODES):
- if os.path.isdir(os.path.join("cache","node"+str(i))):
- shutil.rmtree(os.path.join("cache","node"+str(i)))
+ if os.path.isdir(os.path.join(cachedir,"node"+str(i))):
+ shutil.rmtree(os.path.join(cachedir,"node"+str(i)))
# Create cache directories, run bitcoinds:
for i in range(MAX_NODES):
- datadir=initialize_datadir("cache", i)
+ datadir=initialize_datadir(cachedir, i)
args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ]
if i > 0:
args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
@@ -224,7 +257,7 @@ def initialize_chain(test_dir, num_nodes):
print("initialize_chain: bitcoind started, waiting for RPC to come up")
wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i)
if os.getenv("PYTHON_DEBUG", ""):
- print("initialize_chain: RPC succesfully started")
+ print("initialize_chain: RPC successfully started")
rpcs = []
for i in range(MAX_NODES):
@@ -254,16 +287,15 @@ def initialize_chain(test_dir, num_nodes):
# Shut them down, and clean up cache directories:
stop_nodes(rpcs)
- wait_bitcoinds()
disable_mocktime()
for i in range(MAX_NODES):
- os.remove(log_filename("cache", i, "debug.log"))
- os.remove(log_filename("cache", i, "db.log"))
- os.remove(log_filename("cache", i, "peers.dat"))
- os.remove(log_filename("cache", i, "fee_estimates.dat"))
+ os.remove(log_filename(cachedir, i, "debug.log"))
+ os.remove(log_filename(cachedir, i, "db.log"))
+ os.remove(log_filename(cachedir, i, "peers.dat"))
+ os.remove(log_filename(cachedir, i, "fee_estimates.dat"))
for i in range(num_nodes):
- from_dir = os.path.join("cache", "node"+str(i))
+ from_dir = os.path.join(cachedir, "node"+str(i))
to_dir = os.path.join(test_dir, "node"+str(i))
shutil.copytree(from_dir, to_dir)
initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf
@@ -312,7 +344,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
url = rpc_url(i, rpchost)
wait_for_bitcoind_start(bitcoind_processes[i], url, i)
if os.getenv("PYTHON_DEBUG", ""):
- print("start_node: RPC succesfully started")
+ print("start_node: RPC successfully started")
proxy = get_rpc_proxy(url, i, timeout=timewait)
if COVERAGE_DIR:
@@ -320,7 +352,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
return proxy
-def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
+def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
"""
Start multiple bitcoinds, return RPC connections to them
"""
@@ -329,7 +361,7 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
rpcs = []
try:
for i in range(num_nodes):
- rpcs.append(start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]))
+ rpcs.append(start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i]))
except: # If one node failed to start, stop the others
stop_nodes(rpcs)
raise
@@ -353,6 +385,7 @@ def stop_nodes(nodes):
except http.client.CannotSendRequest as e:
print("WARN: Unable to stop node: " + repr(e))
del nodes[:] # Emptying array closes connections as a side effect
+ wait_bitcoinds()
def set_node_times(nodes, t):
for node in nodes:
@@ -491,19 +524,39 @@ def assert_fee_amount(fee, tx_size, fee_per_kB):
if fee > (tx_size + 2) * fee_per_kB / 1000:
raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)"%(str(fee), str(target_fee)))
-def assert_equal(thing1, thing2):
- if thing1 != thing2:
- raise AssertionError("%s != %s"%(str(thing1),str(thing2)))
+def assert_equal(thing1, thing2, *args):
+ if thing1 != thing2 or any(thing1 != arg for arg in args):
+ raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args))
def assert_greater_than(thing1, thing2):
if thing1 <= thing2:
raise AssertionError("%s <= %s"%(str(thing1),str(thing2)))
+def assert_greater_than_or_equal(thing1, thing2):
+ if thing1 < thing2:
+ raise AssertionError("%s < %s"%(str(thing1),str(thing2)))
+
def assert_raises(exc, fun, *args, **kwds):
+ assert_raises_message(exc, None, fun, *args, **kwds)
+
+def assert_raises_message(exc, message, fun, *args, **kwds):
+ try:
+ fun(*args, **kwds)
+ except exc as e:
+ if message is not None and message not in e.error['message']:
+ raise AssertionError("Expected substring not found:"+e.error['message'])
+ except Exception as e:
+ raise AssertionError("Unexpected exception raised: "+type(e).__name__)
+ else:
+ raise AssertionError("No exception raised")
+
+def assert_raises_jsonrpc(code, fun, *args, **kwds):
+ '''Check for specific JSONRPC exception code'''
try:
fun(*args, **kwds)
- except exc:
- pass
+ except JSONRPCException as e:
+ if e.error["code"] != code:
+ raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"])
except Exception as e:
raise AssertionError("Unexpected exception raised: "+type(e).__name__)
else:
@@ -617,16 +670,15 @@ def create_tx(node, coinbase, to_address, amount):
# Create a spend of each passed-in utxo, splicing in "txouts" to each raw
# transaction to make it large. See gen_return_txouts() above.
-def create_lots_of_big_transactions(node, txouts, utxos, fee):
+def create_lots_of_big_transactions(node, txouts, utxos, num, fee):
addr = node.getnewaddress()
txids = []
- for i in range(len(utxos)):
+ for _ in range(num):
t = utxos.pop()
- inputs = []
- inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
+ inputs=[{ "txid" : t["txid"], "vout" : t["vout"]}]
outputs = {}
- send_value = t['amount'] - fee
- outputs[addr] = satoshi_round(send_value)
+ change = t['amount'] - fee
+ outputs[addr] = satoshi_round(change)
rawtx = node.createrawtransaction(inputs, outputs)
newtx = rawtx[0:92]
newtx = newtx + txouts
@@ -636,6 +688,19 @@ def create_lots_of_big_transactions(node, txouts, utxos, fee):
txids.append(txid)
return txids
+def mine_large_block(node, utxos=None):
+ # generate a 66k transaction,
+ # and 14 of them is close to the 1MB block limit
+ num = 14
+ txouts = gen_return_txouts()
+ utxos = utxos if utxos is not None else []
+ if len(utxos) < num:
+ utxos.clear()
+ utxos.extend(node.listunspent())
+ fee = 100 * node.getnetworkinfo()["relayfee"]
+ create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee)
+ node.generate(1)
+
def get_bip9_status(node, key):
info = node.getblockchaininfo()
return info['bip9_softforks'][key]
generated by cgit v1.2.3 (git 2.26.2) at 2024-11-10 20:57:56 +0000