Add p2p-fullblocktest.py

5 files changed, 286 insertions, 28 deletions

diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py
index c57b6df81b..b9775b477c 100644
--- a/qa/rpc-tests/test_framework/blockstore.py
+++ b/qa/rpc-tests/test_framework/blockstore.py
@@ -10,6 +10,7 @@ class BlockStore(object):
     def __init__(self, datadir):
         self.blockDB = dbm.open(datadir + "/blocks", 'c')
         self.currentBlock = 0L
+        self.headers_map = dict()
     
     def close(self):
         self.blockDB.close()
@@ -26,24 +27,30 @@ class BlockStore(object):
         ret.calc_sha256()
         return ret
 
+    def get_header(self, blockhash):
+        try:
+            return self.headers_map[blockhash]
+        except KeyError:
+            return None
+
     # Note: this pulls full blocks out of the database just to retrieve
     # the headers -- perhaps we could keep a separate data structure
     # to avoid this overhead.
     def headers_for(self, locator, hash_stop, current_tip=None):
         if current_tip is None:
             current_tip = self.currentBlock
-        current_block = self.get(current_tip)
-        if current_block is None:
+        current_block_header = self.get_header(current_tip)
+        if current_block_header is None:
             return None
 
         response = msg_headers()
-        headersList = [ CBlockHeader(current_block) ]
+        headersList = [ current_block_header ]
         maxheaders = 2000
         while (headersList[0].sha256 not in locator.vHave):
             prevBlockHash = headersList[0].hashPrevBlock
-            prevBlock = self.get(prevBlockHash)
-            if prevBlock is not None:
-                headersList.insert(0, CBlockHeader(prevBlock))
+            prevBlockHeader = self.get_header(prevBlockHash)
+            if prevBlockHeader is not None:
+                headersList.insert(0, prevBlockHeader)
             else:
                 break
         headersList = headersList[:maxheaders] # truncate if we have too many
@@ -61,6 +68,10 @@ class BlockStore(object):
         except TypeError as e:
             print "Unexpected error: ", sys.exc_info()[0], e.args
         self.currentBlock = block.sha256
+        self.headers_map[block.sha256] = CBlockHeader(block)
+
+    def add_header(self, header):
+        self.headers_map[header.sha256] = header
 
     def get_blocks(self, inv):
         responses = []
diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py
index f397fe7cd6..59aa8c15cc 100644
--- a/qa/rpc-tests/test_framework/blocktools.py
+++ b/qa/rpc-tests/test_framework/blocktools.py
@@ -5,7 +5,7 @@
 #
 
 from mininode import *
-from script import CScript, CScriptOp
+from script import CScript, CScriptOp, OP_TRUE, OP_CHECKSIG
 
 # Create a block (with regtest difficulty)
 def create_block(hashprev, coinbase, nTime=None):
@@ -37,19 +37,21 @@ def serialize_script_num(value):
         r[-1] |= 0x80
     return r
 
-counter=1
-# Create an anyone-can-spend coinbase transaction, assuming no miner fees
-def create_coinbase(heightAdjust = 0):
-    global counter
+# Create a coinbase transaction, assuming no miner fees.
+# If pubkey is passed in, the coinbase output will be a P2PK output;
+# otherwise an anyone-can-spend output.
+def create_coinbase(height, pubkey = None):
     coinbase = CTransaction()
     coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), 
-                ser_string(serialize_script_num(counter+heightAdjust)), 0xffffffff))
-    counter += 1
+                ser_string(serialize_script_num(height)), 0xffffffff))
     coinbaseoutput = CTxOut()
     coinbaseoutput.nValue = 50*100000000
-    halvings = int((counter+heightAdjust)/150) # regtest
+    halvings = int(height/150) # regtest
     coinbaseoutput.nValue >>= halvings
-    coinbaseoutput.scriptPubKey = ""
+    if (pubkey != None):
+        coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG])
+    else:
+        coinbaseoutput.scriptPubKey = CScript([OP_TRUE])
     coinbase.vout = [ coinbaseoutput ]
     coinbase.calc_sha256()
     return coinbase
diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py
index b945f1bf29..e0b3ce040d 100755
--- a/qa/rpc-tests/test_framework/comptool.py
+++ b/qa/rpc-tests/test_framework/comptool.py
@@ -122,12 +122,19 @@ class TestNode(NodeConnCB):
 # Instances of these are generated by the test generator, and fed into the
 # comptool.
 #
-# "blocks_and_transactions" should be an array of [obj, True/False/None]:
-#  - obj is either a CBlock or a CTransaction, and 
+# "blocks_and_transactions" should be an array of
+#    [obj, True/False/None, hash/None]:
+#  - obj is either a CBlock, CBlockHeader, or a CTransaction, and
 #  - the second value indicates whether the object should be accepted
 #    into the blockchain or mempool (for tests where we expect a certain
 #    answer), or "None" if we don't expect a certain answer and are just
 #    comparing the behavior of the nodes being tested.
+#  - the third value is the hash to test the tip against (if None or omitted,
+#    use the hash of the block)
+#  - NOTE: if a block header, no test is performed; instead the header is
+#    just added to the block_store.  This is to facilitate block delivery
+#    when communicating with headers-first clients (when withholding an
+#    intermediate block).
 # sync_every_block: if True, then each block will be inv'ed, synced, and
 #    nodes will be tested based on the outcome for the block.  If False,
 #    then inv's accumulate until all blocks are processed (or max inv size
@@ -194,7 +201,6 @@ class TestManager(object):
         if not wait_until(blocks_requested, attempts=20*num_blocks):
             # print [ c.cb.block_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested block")
-        # --> Answer request (we did this inline!)
 
         # Send getheaders message
         [ c.cb.send_getheaders() for c in self.connections ]
@@ -217,7 +223,6 @@ class TestManager(object):
         if not wait_until(transaction_requested, attempts=20*num_events):
             # print [ c.cb.tx_request_map for c in self.connections ]
             raise AssertionError("Not all nodes requested transaction")
-        # --> Answer request (we did this inline!)
 
         # Get the mempool
         [ c.cb.send_mempool() for c in self.connections ]
@@ -272,29 +277,55 @@ class TestManager(object):
             # We use these variables to keep track of the last block
             # and last transaction in the tests, which are used
             # if we're not syncing on every block or every tx.
-            [ block, block_outcome ] = [ None, None ]
+            [ block, block_outcome, tip ] = [ None, None, None ]
             [ tx, tx_outcome ] = [ None, None ]
             invqueue = []
 
-            for b_or_t, outcome in test_instance.blocks_and_transactions:
+            for test_obj in test_instance.blocks_and_transactions:
+                b_or_t = test_obj[0]
+                outcome = test_obj[1]
                 # Determine if we're dealing with a block or tx
                 if isinstance(b_or_t, CBlock):  # Block test runner
                     block = b_or_t
                     block_outcome = outcome
+                    tip = block.sha256
+                    # each test_obj can have an optional third argument
+                    # to specify the tip we should compare with
+                    # (default is to use the block being tested)
+                    if len(test_obj) >= 3:
+                        tip = test_obj[2]
+
                     # Add to shared block_store, set as current block
+                    # If there was an open getdata request for the block
+                    # previously, and we didn't have an entry in the
+                    # block_store, then immediately deliver, because the
+                    # node wouldn't send another getdata request while
+                    # the earlier one is outstanding.
+                    first_block_with_hash = True
+                    if self.block_store.get(block.sha256) is not None:
+                        first_block_with_hash = False
                     with mininode_lock:
                         self.block_store.add_block(block)
                         for c in self.connections:
-                            c.cb.block_request_map[block.sha256] = False
+                            if first_block_with_hash and block.sha256 in c.cb.block_request_map and c.cb.block_request_map[block.sha256] == True:
+                                # There was a previous request for this block hash
+                                # Most likely, we delivered a header for this block
+                                # but never had the block to respond to the getdata
+                                c.send_message(msg_block(block))
+                            else:
+                                c.cb.block_request_map[block.sha256] = False
                     # Either send inv's to each node and sync, or add
                     # to invqueue for later inv'ing.
                     if (test_instance.sync_every_block):
                         [ c.cb.send_inv(block) for c in self.connections ]
                         self.sync_blocks(block.sha256, 1)
-                        if (not self.check_results(block.sha256, outcome)):
+                        if (not self.check_results(tip, outcome)):
                             raise AssertionError("Test failed at test %d" % test_number)
                     else:
                         invqueue.append(CInv(2, block.sha256))
+                elif isinstance(b_or_t, CBlockHeader):
+                    block_header = b_or_t
+                    self.block_store.add_header(block_header)
                 else:  # Tx test runner
                     assert(isinstance(b_or_t, CTransaction))
                     tx = b_or_t
@@ -322,9 +353,8 @@ class TestManager(object):
                 if len(invqueue) > 0:
                     [ c.send_message(msg_inv(invqueue)) for c in self.connections ]
                     invqueue = []
-                self.sync_blocks(block.sha256,
-                                 len(test_instance.blocks_and_transactions))
-                if (not self.check_results(block.sha256, block_outcome)):
+                self.sync_blocks(block.sha256, len(test_instance.blocks_and_transactions))
+                if (not self.check_results(tip, block_outcome)):
                     raise AssertionError("Block test failed at test %d" % test_number)
             if (not test_instance.sync_every_tx and tx is not None):
                 if len(invqueue) > 0:
diff --git a/qa/rpc-tests/test_framework/key.py b/qa/rpc-tests/test_framework/key.py
new file mode 100644
index 0000000000..ba3038fe04
--- /dev/null
+++ b/qa/rpc-tests/test_framework/key.py
@@ -0,0 +1,215 @@
+# Copyright (c) 2011 Sam Rushing
+#
+# key.py - OpenSSL wrapper
+#
+# This file is modified from python-bitcoinlib.
+#
+
+"""ECC secp256k1 crypto routines
+
+WARNING: This module does not mlock() secrets; your private keys may end up on
+disk in swap! Use with caution!
+"""
+
+import ctypes
+import ctypes.util
+import hashlib
+import sys
+
+ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library ('ssl') or 'libeay32')
+
+ssl.BN_new.restype = ctypes.c_void_p
+ssl.BN_new.argtypes = []
+
+ssl.BN_bin2bn.restype = ctypes.c_void_p
+ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p]
+
+ssl.BN_CTX_free.restype = None
+ssl.BN_CTX_free.argtypes = [ctypes.c_void_p]
+
+ssl.BN_CTX_new.restype = ctypes.c_void_p
+ssl.BN_CTX_new.argtypes = []
+
+ssl.ECDH_compute_key.restype = ctypes.c_int
+ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.ECDSA_sign.restype = ctypes.c_int
+ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.ECDSA_verify.restype = ctypes.c_int
+ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p]
+
+ssl.EC_KEY_free.restype = None
+ssl.EC_KEY_free.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
+ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int]
+
+ssl.EC_KEY_get0_group.restype = ctypes.c_void_p
+ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p
+ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p]
+
+ssl.EC_KEY_set_private_key.restype = ctypes.c_int
+ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.EC_KEY_set_conv_form.restype = None
+ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int]
+
+ssl.EC_KEY_set_public_key.restype = ctypes.c_int
+ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.i2o_ECPublicKey.restype = ctypes.c_void_p
+ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+
+ssl.EC_POINT_new.restype = ctypes.c_void_p
+ssl.EC_POINT_new.argtypes = [ctypes.c_void_p]
+
+ssl.EC_POINT_free.restype = None
+ssl.EC_POINT_free.argtypes = [ctypes.c_void_p]
+
+ssl.EC_POINT_mul.restype = ctypes.c_int
+ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+
+# this specifies the curve used with ECDSA.
+NID_secp256k1 = 714 # from openssl/obj_mac.h
+
+# Thx to Sam Devlin for the ctypes magic 64-bit fix.
+def _check_result(val, func, args):
+    if val == 0:
+        raise ValueError
+    else:
+        return ctypes.c_void_p (val)
+
+ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
+ssl.EC_KEY_new_by_curve_name.errcheck = _check_result
+
+class CECKey(object):
+    """Wrapper around OpenSSL's EC_KEY"""
+
+    POINT_CONVERSION_COMPRESSED = 2
+    POINT_CONVERSION_UNCOMPRESSED = 4
+
+    def __init__(self):
+        self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1)
+
+    def __del__(self):
+        if ssl:
+            ssl.EC_KEY_free(self.k)
+        self.k = None
+
+    def set_secretbytes(self, secret):
+        priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new())
+        group = ssl.EC_KEY_get0_group(self.k)
+        pub_key = ssl.EC_POINT_new(group)
+        ctx = ssl.BN_CTX_new()
+        if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx):
+            raise ValueError("Could not derive public key from the supplied secret.")
+        ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx)
+        ssl.EC_KEY_set_private_key(self.k, priv_key)
+        ssl.EC_KEY_set_public_key(self.k, pub_key)
+        ssl.EC_POINT_free(pub_key)
+        ssl.BN_CTX_free(ctx)
+        return self.k
+
+    def set_privkey(self, key):
+        self.mb = ctypes.create_string_buffer(key)
+        return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key))
+
+    def set_pubkey(self, key):
+        self.mb = ctypes.create_string_buffer(key)
+        return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key))
+
+    def get_privkey(self):
+        size = ssl.i2d_ECPrivateKey(self.k, 0)
+        mb_pri = ctypes.create_string_buffer(size)
+        ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri)))
+        return mb_pri.raw
+
+    def get_pubkey(self):
+        size = ssl.i2o_ECPublicKey(self.k, 0)
+        mb = ctypes.create_string_buffer(size)
+        ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb)))
+        return mb.raw
+
+    def get_raw_ecdh_key(self, other_pubkey):
+        ecdh_keybuffer = ctypes.create_string_buffer(32)
+        r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32,
+                                 ssl.EC_KEY_get0_public_key(other_pubkey.k),
+                                 self.k, 0)
+        if r != 32:
+            raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed')
+        return ecdh_keybuffer.raw
+
+    def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()):
+        # FIXME: be warned it's not clear what the kdf should be as a default
+        r = self.get_raw_ecdh_key(other_pubkey)
+        return kdf(r)
+
+    def sign(self, hash):
+        # FIXME: need unit tests for below cases
+        if not isinstance(hash, bytes):
+            raise TypeError('Hash must be bytes instance; got %r' % hash.__class__)
+        if len(hash) != 32:
+            raise ValueError('Hash must be exactly 32 bytes long')
+
+        sig_size0 = ctypes.c_uint32()
+        sig_size0.value = ssl.ECDSA_size(self.k)
+        mb_sig = ctypes.create_string_buffer(sig_size0.value)
+        result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k)
+        assert 1 == result
+        return mb_sig.raw[:sig_size0.value]
+
+    def verify(self, hash, sig):
+        """Verify a DER signature"""
+        return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1
+
+    def set_compressed(self, compressed):
+        if compressed:
+            form = self.POINT_CONVERSION_COMPRESSED
+        else:
+            form = self.POINT_CONVERSION_UNCOMPRESSED
+        ssl.EC_KEY_set_conv_form(self.k, form)
+
+
+class CPubKey(bytes):
+    """An encapsulated public key
+
+    Attributes:
+
+    is_valid      - Corresponds to CPubKey.IsValid()
+    is_fullyvalid - Corresponds to CPubKey.IsFullyValid()
+    is_compressed - Corresponds to CPubKey.IsCompressed()
+    """
+
+    def __new__(cls, buf, _cec_key=None):
+        self = super(CPubKey, cls).__new__(cls, buf)
+        if _cec_key is None:
+            _cec_key = CECKey()
+        self._cec_key = _cec_key
+        self.is_fullyvalid = _cec_key.set_pubkey(self) != 0
+        return self
+
+    @property
+    def is_valid(self):
+        return len(self) > 0
+
+    @property
+    def is_compressed(self):
+        return len(self) == 33
+
+    def verify(self, hash, sig):
+        return self._cec_key.verify(hash, sig)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        # Always have represent as b'<secret>' so test cases don't have to
+        # change for py2/3
+        if sys.version > '3':
+            return '%s(%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__())
+        else:
+            return '%s(b%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__())
+
diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py
index e37ab5d45a..0a78cf6fb1 100644
--- a/qa/rpc-tests/test_framework/script.py
+++ b/qa/rpc-tests/test_framework/script.py
@@ -27,7 +27,7 @@ if sys.version > '3':
 import copy
 import struct
 
-import test_framework.bignum
+from test_framework.bignum import bn2vch
 
 MAX_SCRIPT_SIZE = 10000
 MAX_SCRIPT_ELEMENT_SIZE = 520
@@ -664,7 +664,7 @@ class CScript(bytes):
             elif other == -1:
                 other = bytes(bchr(OP_1NEGATE))
             else:
-                other = CScriptOp.encode_op_pushdata(bignum.bn2vch(other))
+                other = CScriptOp.encode_op_pushdata(bn2vch(other))
         elif isinstance(other, (bytes, bytearray)):
             other = CScriptOp.encode_op_pushdata(other)
         return other