From f9cd9b1bfaf65b4e24c1eb9ca8268513e78cc458 Mon Sep 17 00:00:00 2001
From: John Newbery <john@johnnewbery.com>
Date: Mon, 16 Oct 2017 22:25:15 -0400
Subject: [tests] Move test_framework Bitcoin primitives into separate module

mininode.py wildcard imports all names from primitives.py. This is
to avoid having to change all test scripts that import from mininode.py.
---
 test/functional/test_framework/mininode.py | 1306 +---------------------------
 1 file changed, 1 insertion(+), 1305 deletions(-)

(limited to 'test/functional/test_framework/mininode.py')

diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py
index 8fbc63fba4..2a1ac2d705 100755
--- a/test/functional/test_framework/mininode.py
+++ b/test/functional/test_framework/mininode.py
@@ -12,47 +12,18 @@ found in the mini-node branch of http://github.com/jgarzik/pynode.
 NodeConn: an object which manages p2p connectivity to a bitcoin node
 NodeConnCB: a base class that describes the interface for receiving
             callbacks with network messages from a NodeConn
-CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....:
-    data structures that should map to corresponding structures in
-    bitcoin/primitives
-msg_block, msg_tx, msg_headers, etc.:
-    data structures that represent network messages
-ser_*, deser_*: functions that handle serialization/deserialization
 """
-
 import asyncore
-from codecs import encode
 from collections import defaultdict
-import copy
-import hashlib
 from io import BytesIO
 import logging
-import random
 import socket
 import struct
 import sys
 import time
 from threading import RLock, Thread
 
-from test_framework.siphash import siphash256
-from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until
-
-MIN_VERSION_SUPPORTED = 60001
-MY_VERSION = 70014  # past bip-31 for ping/pong
-MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
-MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
-
-MAX_INV_SZ = 50000
-MAX_BLOCK_BASE_SIZE = 1000000
-
-COIN = 100000000 # 1 btc in satoshis
-
-NODE_NETWORK = (1 << 0)
-# NODE_GETUTXO = (1 << 1)
-# NODE_BLOOM = (1 << 2)
-NODE_WITNESS = (1 << 3)
-NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5)
-NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7)
+from test_framework.messages import *
 
 logger = logging.getLogger("TestFramework.mininode")
 
@@ -69,1281 +40,6 @@ mininode_socket_map = dict()
 # access to any data shared with the NodeConnCB or NodeConn.
 mininode_lock = RLock()
 
-# Serialization/deserialization tools
-def sha256(s):
-    return hashlib.new('sha256', s).digest()
-
-def ripemd160(s):
-    return hashlib.new('ripemd160', s).digest()
-
-def hash256(s):
-    return sha256(sha256(s))
-
-def ser_compact_size(l):
-    r = b""
-    if l < 253:
-        r = struct.pack("B", l)
-    elif l < 0x10000:
-        r = struct.pack("<BH", 253, l)
-    elif l < 0x100000000:
-        r = struct.pack("<BI", 254, l)
-    else:
-        r = struct.pack("<BQ", 255, l)
-    return r
-
-def deser_compact_size(f):
-    nit = struct.unpack("<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack("<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack("<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack("<Q", f.read(8))[0]
-    return nit
-
-def deser_string(f):
-    nit = deser_compact_size(f)
-    return f.read(nit)
-
-def ser_string(s):
-    return ser_compact_size(len(s)) + s
-
-def deser_uint256(f):
-    r = 0
-    for i in range(8):
-        t = struct.unpack("<I", f.read(4))[0]
-        r += t << (i * 32)
-    return r
-
-
-def ser_uint256(u):
-    rs = b""
-    for i in range(8):
-        rs += struct.pack("<I", u & 0xFFFFFFFF)
-        u >>= 32
-    return rs
-
-
-def uint256_from_str(s):
-    r = 0
-    t = struct.unpack("<IIIIIIII", s[:32])
-    for i in range(8):
-        r += t[i] << (i * 32)
-    return r
-
-
-def uint256_from_compact(c):
-    nbytes = (c >> 24) & 0xFF
-    v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
-    return v
-
-
-def deser_vector(f, c):
-    nit = deser_compact_size(f)
-    r = []
-    for i in range(nit):
-        t = c()
-        t.deserialize(f)
-        r.append(t)
-    return r
-
-
-# ser_function_name: Allow for an alternate serialization function on the
-# entries in the vector (we use this for serializing the vector of transactions
-# for a witness block).
-def ser_vector(l, ser_function_name=None):
-    r = ser_compact_size(len(l))
-    for i in l:
-        if ser_function_name:
-            r += getattr(i, ser_function_name)()
-        else:
-            r += i.serialize()
-    return r
-
-
-def deser_uint256_vector(f):
-    nit = deser_compact_size(f)
-    r = []
-    for i in range(nit):
-        t = deser_uint256(f)
-        r.append(t)
-    return r
-
-
-def ser_uint256_vector(l):
-    r = ser_compact_size(len(l))
-    for i in l:
-        r += ser_uint256(i)
-    return r
-
-
-def deser_string_vector(f):
-    nit = deser_compact_size(f)
-    r = []
-    for i in range(nit):
-        t = deser_string(f)
-        r.append(t)
-    return r
-
-
-def ser_string_vector(l):
-    r = ser_compact_size(len(l))
-    for sv in l:
-        r += ser_string(sv)
-    return r
-
-
-def deser_int_vector(f):
-    nit = deser_compact_size(f)
-    r = []
-    for i in range(nit):
-        t = struct.unpack("<i", f.read(4))[0]
-        r.append(t)
-    return r
-
-
-def ser_int_vector(l):
-    r = ser_compact_size(len(l))
-    for i in l:
-        r += struct.pack("<i", i)
-    return r
-
-# Deserialize from a hex string representation (eg from RPC)
-def FromHex(obj, hex_string):
-    obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
-    return obj
-
-# Convert a binary-serializable object to hex (eg for submission via RPC)
-def ToHex(obj):
-    return bytes_to_hex_str(obj.serialize())
-
-# Objects that map to bitcoind objects, which can be serialized/deserialized
-
-class CAddress():
-    def __init__(self):
-        self.nServices = 1
-        self.pchReserved = b"\x00" * 10 + b"\xff" * 2
-        self.ip = "0.0.0.0"
-        self.port = 0
-
-    def deserialize(self, f):
-        self.nServices = struct.unpack("<Q", f.read(8))[0]
-        self.pchReserved = f.read(12)
-        self.ip = socket.inet_ntoa(f.read(4))
-        self.port = struct.unpack(">H", f.read(2))[0]
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<Q", self.nServices)
-        r += self.pchReserved
-        r += socket.inet_aton(self.ip)
-        r += struct.pack(">H", self.port)
-        return r
-
-    def __repr__(self):
-        return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
-                                                         self.ip, self.port)
-
-MSG_WITNESS_FLAG = 1<<30
-
-class CInv():
-    typemap = {
-        0: "Error",
-        1: "TX",
-        2: "Block",
-        1|MSG_WITNESS_FLAG: "WitnessTx",
-        2|MSG_WITNESS_FLAG : "WitnessBlock",
-        4: "CompactBlock"
-    }
-
-    def __init__(self, t=0, h=0):
-        self.type = t
-        self.hash = h
-
-    def deserialize(self, f):
-        self.type = struct.unpack("<i", f.read(4))[0]
-        self.hash = deser_uint256(f)
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<i", self.type)
-        r += ser_uint256(self.hash)
-        return r
-
-    def __repr__(self):
-        return "CInv(type=%s hash=%064x)" \
-            % (self.typemap[self.type], self.hash)
-
-
-class CBlockLocator():
-    def __init__(self):
-        self.nVersion = MY_VERSION
-        self.vHave = []
-
-    def deserialize(self, f):
-        self.nVersion = struct.unpack("<i", f.read(4))[0]
-        self.vHave = deser_uint256_vector(f)
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<i", self.nVersion)
-        r += ser_uint256_vector(self.vHave)
-        return r
-
-    def __repr__(self):
-        return "CBlockLocator(nVersion=%i vHave=%s)" \
-            % (self.nVersion, repr(self.vHave))
-
-
-class COutPoint():
-    def __init__(self, hash=0, n=0):
-        self.hash = hash
-        self.n = n
-
-    def deserialize(self, f):
-        self.hash = deser_uint256(f)
-        self.n = struct.unpack("<I", f.read(4))[0]
-
-    def serialize(self):
-        r = b""
-        r += ser_uint256(self.hash)
-        r += struct.pack("<I", self.n)
-        return r
-
-    def __repr__(self):
-        return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n)
-
-
-class CTxIn():
-    def __init__(self, outpoint=None, scriptSig=b"", nSequence=0):
-        if outpoint is None:
-            self.prevout = COutPoint()
-        else:
-            self.prevout = outpoint
-        self.scriptSig = scriptSig
-        self.nSequence = nSequence
-
-    def deserialize(self, f):
-        self.prevout = COutPoint()
-        self.prevout.deserialize(f)
-        self.scriptSig = deser_string(f)
-        self.nSequence = struct.unpack("<I", f.read(4))[0]
-
-    def serialize(self):
-        r = b""
-        r += self.prevout.serialize()
-        r += ser_string(self.scriptSig)
-        r += struct.pack("<I", self.nSequence)
-        return r
-
-    def __repr__(self):
-        return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
-            % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
-               self.nSequence)
-
-
-class CTxOut():
-    def __init__(self, nValue=0, scriptPubKey=b""):
-        self.nValue = nValue
-        self.scriptPubKey = scriptPubKey
-
-    def deserialize(self, f):
-        self.nValue = struct.unpack("<q", f.read(8))[0]
-        self.scriptPubKey = deser_string(f)
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<q", self.nValue)
-        r += ser_string(self.scriptPubKey)
-        return r
-
-    def __repr__(self):
-        return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
-            % (self.nValue // COIN, self.nValue % COIN,
-               bytes_to_hex_str(self.scriptPubKey))
-
-
-class CScriptWitness():
-    def __init__(self):
-        # stack is a vector of strings
-        self.stack = []
-
-    def __repr__(self):
-        return "CScriptWitness(%s)" % \
-               (",".join([bytes_to_hex_str(x) for x in self.stack]))
-
-    def is_null(self):
-        if self.stack:
-            return False
-        return True
-
-
-class CTxInWitness():
-    def __init__(self):
-        self.scriptWitness = CScriptWitness()
-
-    def deserialize(self, f):
-        self.scriptWitness.stack = deser_string_vector(f)
-
-    def serialize(self):
-        return ser_string_vector(self.scriptWitness.stack)
-
-    def __repr__(self):
-        return repr(self.scriptWitness)
-
-    def is_null(self):
-        return self.scriptWitness.is_null()
-
-
-class CTxWitness():
-    def __init__(self):
-        self.vtxinwit = []
-
-    def deserialize(self, f):
-        for i in range(len(self.vtxinwit)):
-            self.vtxinwit[i].deserialize(f)
-
-    def serialize(self):
-        r = b""
-        # This is different than the usual vector serialization --
-        # we omit the length of the vector, which is required to be
-        # the same length as the transaction's vin vector.
-        for x in self.vtxinwit:
-            r += x.serialize()
-        return r
-
-    def __repr__(self):
-        return "CTxWitness(%s)" % \
-               (';'.join([repr(x) for x in self.vtxinwit]))
-
-    def is_null(self):
-        for x in self.vtxinwit:
-            if not x.is_null():
-                return False
-        return True
-
-
-class CTransaction():
-    def __init__(self, tx=None):
-        if tx is None:
-            self.nVersion = 1
-            self.vin = []
-            self.vout = []
-            self.wit = CTxWitness()
-            self.nLockTime = 0
-            self.sha256 = None
-            self.hash = None
-        else:
-            self.nVersion = tx.nVersion
-            self.vin = copy.deepcopy(tx.vin)
-            self.vout = copy.deepcopy(tx.vout)
-            self.nLockTime = tx.nLockTime
-            self.sha256 = tx.sha256
-            self.hash = tx.hash
-            self.wit = copy.deepcopy(tx.wit)
-
-    def deserialize(self, f):
-        self.nVersion = struct.unpack("<i", f.read(4))[0]
-        self.vin = deser_vector(f, CTxIn)
-        flags = 0
-        if len(self.vin) == 0:
-            flags = struct.unpack("<B", f.read(1))[0]
-            # Not sure why flags can't be zero, but this
-            # matches the implementation in bitcoind
-            if (flags != 0):
-                self.vin = deser_vector(f, CTxIn)
-                self.vout = deser_vector(f, CTxOut)
-        else:
-            self.vout = deser_vector(f, CTxOut)
-        if flags != 0:
-            self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
-            self.wit.deserialize(f)
-        self.nLockTime = struct.unpack("<I", f.read(4))[0]
-        self.sha256 = None
-        self.hash = None
-
-    def serialize_without_witness(self):
-        r = b""
-        r += struct.pack("<i", self.nVersion)
-        r += ser_vector(self.vin)
-        r += ser_vector(self.vout)
-        r += struct.pack("<I", self.nLockTime)
-        return r
-
-    # Only serialize with witness when explicitly called for
-    def serialize_with_witness(self):
-        flags = 0
-        if not self.wit.is_null():
-            flags |= 1
-        r = b""
-        r += struct.pack("<i", self.nVersion)
-        if flags:
-            dummy = []
-            r += ser_vector(dummy)
-            r += struct.pack("<B", flags)
-        r += ser_vector(self.vin)
-        r += ser_vector(self.vout)
-        if flags & 1:
-            if (len(self.wit.vtxinwit) != len(self.vin)):
-                # vtxinwit must have the same length as vin
-                self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
-                for i in range(len(self.wit.vtxinwit), len(self.vin)):
-                    self.wit.vtxinwit.append(CTxInWitness())
-            r += self.wit.serialize()
-        r += struct.pack("<I", self.nLockTime)
-        return r
-
-    # Regular serialization is without witness -- must explicitly
-    # call serialize_with_witness to include witness data.
-    def serialize(self):
-        return self.serialize_without_witness()
-
-    # Recalculate the txid (transaction hash without witness)
-    def rehash(self):
-        self.sha256 = None
-        self.calc_sha256()
-
-    # We will only cache the serialization without witness in
-    # self.sha256 and self.hash -- those are expected to be the txid.
-    def calc_sha256(self, with_witness=False):
-        if with_witness:
-            # Don't cache the result, just return it
-            return uint256_from_str(hash256(self.serialize_with_witness()))
-
-        if self.sha256 is None:
-            self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
-        self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
-
-    def is_valid(self):
-        self.calc_sha256()
-        for tout in self.vout:
-            if tout.nValue < 0 or tout.nValue > 21000000 * COIN:
-                return False
-        return True
-
-    def __repr__(self):
-        return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
-            % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
-
-
-class CBlockHeader():
-    def __init__(self, header=None):
-        if header is None:
-            self.set_null()
-        else:
-            self.nVersion = header.nVersion
-            self.hashPrevBlock = header.hashPrevBlock
-            self.hashMerkleRoot = header.hashMerkleRoot
-            self.nTime = header.nTime
-            self.nBits = header.nBits
-            self.nNonce = header.nNonce
-            self.sha256 = header.sha256
-            self.hash = header.hash
-            self.calc_sha256()
-
-    def set_null(self):
-        self.nVersion = 1
-        self.hashPrevBlock = 0
-        self.hashMerkleRoot = 0
-        self.nTime = 0
-        self.nBits = 0
-        self.nNonce = 0
-        self.sha256 = None
-        self.hash = None
-
-    def deserialize(self, f):
-        self.nVersion = struct.unpack("<i", f.read(4))[0]
-        self.hashPrevBlock = deser_uint256(f)
-        self.hashMerkleRoot = deser_uint256(f)
-        self.nTime = struct.unpack("<I", f.read(4))[0]
-        self.nBits = struct.unpack("<I", f.read(4))[0]
-        self.nNonce = struct.unpack("<I", f.read(4))[0]
-        self.sha256 = None
-        self.hash = None
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<i", self.nVersion)
-        r += ser_uint256(self.hashPrevBlock)
-        r += ser_uint256(self.hashMerkleRoot)
-        r += struct.pack("<I", self.nTime)
-        r += struct.pack("<I", self.nBits)
-        r += struct.pack("<I", self.nNonce)
-        return r
-
-    def calc_sha256(self):
-        if self.sha256 is None:
-            r = b""
-            r += struct.pack("<i", self.nVersion)
-            r += ser_uint256(self.hashPrevBlock)
-            r += ser_uint256(self.hashMerkleRoot)
-            r += struct.pack("<I", self.nTime)
-            r += struct.pack("<I", self.nBits)
-            r += struct.pack("<I", self.nNonce)
-            self.sha256 = uint256_from_str(hash256(r))
-            self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii')
-
-    def rehash(self):
-        self.sha256 = None
-        self.calc_sha256()
-        return self.sha256
-
-    def __repr__(self):
-        return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
-            % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
-               time.ctime(self.nTime), self.nBits, self.nNonce)
-
-
-class CBlock(CBlockHeader):
-    def __init__(self, header=None):
-        super(CBlock, self).__init__(header)
-        self.vtx = []
-
-    def deserialize(self, f):
-        super(CBlock, self).deserialize(f)
-        self.vtx = deser_vector(f, CTransaction)
-
-    def serialize(self, with_witness=False):
-        r = b""
-        r += super(CBlock, self).serialize()
-        if with_witness:
-            r += ser_vector(self.vtx, "serialize_with_witness")
-        else:
-            r += ser_vector(self.vtx)
-        return r
-
-    # Calculate the merkle root given a vector of transaction hashes
-    @classmethod
-    def get_merkle_root(cls, hashes):
-        while len(hashes) > 1:
-            newhashes = []
-            for i in range(0, len(hashes), 2):
-                i2 = min(i+1, len(hashes)-1)
-                newhashes.append(hash256(hashes[i] + hashes[i2]))
-            hashes = newhashes
-        return uint256_from_str(hashes[0])
-
-    def calc_merkle_root(self):
-        hashes = []
-        for tx in self.vtx:
-            tx.calc_sha256()
-            hashes.append(ser_uint256(tx.sha256))
-        return self.get_merkle_root(hashes)
-
-    def calc_witness_merkle_root(self):
-        # For witness root purposes, the hash of the
-        # coinbase, with witness, is defined to be 0...0
-        hashes = [ser_uint256(0)]
-
-        for tx in self.vtx[1:]:
-            # Calculate the hashes with witness data
-            hashes.append(ser_uint256(tx.calc_sha256(True)))
-
-        return self.get_merkle_root(hashes)
-
-    def is_valid(self):
-        self.calc_sha256()
-        target = uint256_from_compact(self.nBits)
-        if self.sha256 > target:
-            return False
-        for tx in self.vtx:
-            if not tx.is_valid():
-                return False
-        if self.calc_merkle_root() != self.hashMerkleRoot:
-            return False
-        return True
-
-    def solve(self):
-        self.rehash()
-        target = uint256_from_compact(self.nBits)
-        while self.sha256 > target:
-            self.nNonce += 1
-            self.rehash()
-
-    def __repr__(self):
-        return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
-            % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
-               time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
-
-
-class PrefilledTransaction():
-    def __init__(self, index=0, tx = None):
-        self.index = index
-        self.tx = tx
-
-    def deserialize(self, f):
-        self.index = deser_compact_size(f)
-        self.tx = CTransaction()
-        self.tx.deserialize(f)
-
-    def serialize(self, with_witness=False):
-        r = b""
-        r += ser_compact_size(self.index)
-        if with_witness:
-            r += self.tx.serialize_with_witness()
-        else:
-            r += self.tx.serialize_without_witness()
-        return r
-
-    def serialize_with_witness(self):
-        return self.serialize(with_witness=True)
-
-    def __repr__(self):
-        return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
-
-# This is what we send on the wire, in a cmpctblock message.
-class P2PHeaderAndShortIDs():
-    def __init__(self):
-        self.header = CBlockHeader()
-        self.nonce = 0
-        self.shortids_length = 0
-        self.shortids = []
-        self.prefilled_txn_length = 0
-        self.prefilled_txn = []
-
-    def deserialize(self, f):
-        self.header.deserialize(f)
-        self.nonce = struct.unpack("<Q", f.read(8))[0]
-        self.shortids_length = deser_compact_size(f)
-        for i in range(self.shortids_length):
-            # shortids are defined to be 6 bytes in the spec, so append
-            # two zero bytes and read it in as an 8-byte number
-            self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
-        self.prefilled_txn = deser_vector(f, PrefilledTransaction)
-        self.prefilled_txn_length = len(self.prefilled_txn)
-
-    # When using version 2 compact blocks, we must serialize with_witness.
-    def serialize(self, with_witness=False):
-        r = b""
-        r += self.header.serialize()
-        r += struct.pack("<Q", self.nonce)
-        r += ser_compact_size(self.shortids_length)
-        for x in self.shortids:
-            # We only want the first 6 bytes
-            r += struct.pack("<Q", x)[0:6]
-        if with_witness:
-            r += ser_vector(self.prefilled_txn, "serialize_with_witness")
-        else:
-            r += ser_vector(self.prefilled_txn)
-        return r
-
-    def __repr__(self):
-        return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
-
-# P2P version of the above that will use witness serialization (for compact
-# block version 2)
-class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
-    def serialize(self):
-        return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
-
-# Calculate the BIP 152-compact blocks shortid for a given transaction hash
-def calculate_shortid(k0, k1, tx_hash):
-    expected_shortid = siphash256(k0, k1, tx_hash)
-    expected_shortid &= 0x0000ffffffffffff
-    return expected_shortid
-
-# This version gets rid of the array lengths, and reinterprets the differential
-# encoding into indices that can be used for lookup.
-class HeaderAndShortIDs():
-    def __init__(self, p2pheaders_and_shortids = None):
-        self.header = CBlockHeader()
-        self.nonce = 0
-        self.shortids = []
-        self.prefilled_txn = []
-        self.use_witness = False
-
-        if p2pheaders_and_shortids != None:
-            self.header = p2pheaders_and_shortids.header
-            self.nonce = p2pheaders_and_shortids.nonce
-            self.shortids = p2pheaders_and_shortids.shortids
-            last_index = -1
-            for x in p2pheaders_and_shortids.prefilled_txn:
-                self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
-                last_index = self.prefilled_txn[-1].index
-
-    def to_p2p(self):
-        if self.use_witness:
-            ret = P2PHeaderAndShortWitnessIDs()
-        else:
-            ret = P2PHeaderAndShortIDs()
-        ret.header = self.header
-        ret.nonce = self.nonce
-        ret.shortids_length = len(self.shortids)
-        ret.shortids = self.shortids
-        ret.prefilled_txn_length = len(self.prefilled_txn)
-        ret.prefilled_txn = []
-        last_index = -1
-        for x in self.prefilled_txn:
-            ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
-            last_index = x.index
-        return ret
-
-    def get_siphash_keys(self):
-        header_nonce = self.header.serialize()
-        header_nonce += struct.pack("<Q", self.nonce)
-        hash_header_nonce_as_str = sha256(header_nonce)
-        key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
-        key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
-        return [ key0, key1 ]
-
-    # Version 2 compact blocks use wtxid in shortids (rather than txid)
-    def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
-        self.header = CBlockHeader(block)
-        self.nonce = nonce
-        self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
-        self.shortids = []
-        self.use_witness = use_witness
-        [k0, k1] = self.get_siphash_keys()
-        for i in range(len(block.vtx)):
-            if i not in prefill_list:
-                tx_hash = block.vtx[i].sha256
-                if use_witness:
-                    tx_hash = block.vtx[i].calc_sha256(with_witness=True)
-                self.shortids.append(calculate_shortid(k0, k1, tx_hash))
-
-    def __repr__(self):
-        return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
-
-
-class BlockTransactionsRequest():
-
-    def __init__(self, blockhash=0, indexes = None):
-        self.blockhash = blockhash
-        self.indexes = indexes if indexes != None else []
-
-    def deserialize(self, f):
-        self.blockhash = deser_uint256(f)
-        indexes_length = deser_compact_size(f)
-        for i in range(indexes_length):
-            self.indexes.append(deser_compact_size(f))
-
-    def serialize(self):
-        r = b""
-        r += ser_uint256(self.blockhash)
-        r += ser_compact_size(len(self.indexes))
-        for x in self.indexes:
-            r += ser_compact_size(x)
-        return r
-
-    # helper to set the differentially encoded indexes from absolute ones
-    def from_absolute(self, absolute_indexes):
-        self.indexes = []
-        last_index = -1
-        for x in absolute_indexes:
-            self.indexes.append(x-last_index-1)
-            last_index = x
-
-    def to_absolute(self):
-        absolute_indexes = []
-        last_index = -1
-        for x in self.indexes:
-            absolute_indexes.append(x+last_index+1)
-            last_index = absolute_indexes[-1]
-        return absolute_indexes
-
-    def __repr__(self):
-        return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
-
-
-class BlockTransactions():
-
-    def __init__(self, blockhash=0, transactions = None):
-        self.blockhash = blockhash
-        self.transactions = transactions if transactions != None else []
-
-    def deserialize(self, f):
-        self.blockhash = deser_uint256(f)
-        self.transactions = deser_vector(f, CTransaction)
-
-    def serialize(self, with_witness=False):
-        r = b""
-        r += ser_uint256(self.blockhash)
-        if with_witness:
-            r += ser_vector(self.transactions, "serialize_with_witness")
-        else:
-            r += ser_vector(self.transactions)
-        return r
-
-    def __repr__(self):
-        return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
-
-
-# Objects that correspond to messages on the wire
-class msg_version():
-    command = b"version"
-
-    def __init__(self):
-        self.nVersion = MY_VERSION
-        self.nServices = NODE_NETWORK | NODE_WITNESS
-        self.nTime = int(time.time())
-        self.addrTo = CAddress()
-        self.addrFrom = CAddress()
-        self.nNonce = random.getrandbits(64)
-        self.strSubVer = MY_SUBVERSION
-        self.nStartingHeight = -1
-        self.nRelay = MY_RELAY
-
-    def deserialize(self, f):
-        self.nVersion = struct.unpack("<i", f.read(4))[0]
-        if self.nVersion == 10300:
-            self.nVersion = 300
-        self.nServices = struct.unpack("<Q", f.read(8))[0]
-        self.nTime = struct.unpack("<q", f.read(8))[0]
-        self.addrTo = CAddress()
-        self.addrTo.deserialize(f)
-
-        if self.nVersion >= 106:
-            self.addrFrom = CAddress()
-            self.addrFrom.deserialize(f)
-            self.nNonce = struct.unpack("<Q", f.read(8))[0]
-            self.strSubVer = deser_string(f)
-        else:
-            self.addrFrom = None
-            self.nNonce = None
-            self.strSubVer = None
-            self.nStartingHeight = None
-
-        if self.nVersion >= 209:
-            self.nStartingHeight = struct.unpack("<i", f.read(4))[0]
-        else:
-            self.nStartingHeight = None
-
-        if self.nVersion >= 70001:
-            # Relay field is optional for version 70001 onwards
-            try:
-                self.nRelay = struct.unpack("<b", f.read(1))[0]
-            except:
-                self.nRelay = 0
-        else:
-            self.nRelay = 0
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<i", self.nVersion)
-        r += struct.pack("<Q", self.nServices)
-        r += struct.pack("<q", self.nTime)
-        r += self.addrTo.serialize()
-        r += self.addrFrom.serialize()
-        r += struct.pack("<Q", self.nNonce)
-        r += ser_string(self.strSubVer)
-        r += struct.pack("<i", self.nStartingHeight)
-        r += struct.pack("<b", self.nRelay)
-        return r
-
-    def __repr__(self):
-        return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \
-            % (self.nVersion, self.nServices, time.ctime(self.nTime),
-               repr(self.addrTo), repr(self.addrFrom), self.nNonce,
-               self.strSubVer, self.nStartingHeight, self.nRelay)
-
-
-class msg_verack():
-    command = b"verack"
-
-    def __init__(self):
-        pass
-
-    def deserialize(self, f):
-        pass
-
-    def serialize(self):
-        return b""
-
-    def __repr__(self):
-        return "msg_verack()"
-
-
-class msg_addr():
-    command = b"addr"
-
-    def __init__(self):
-        self.addrs = []
-
-    def deserialize(self, f):
-        self.addrs = deser_vector(f, CAddress)
-
-    def serialize(self):
-        return ser_vector(self.addrs)
-
-    def __repr__(self):
-        return "msg_addr(addrs=%s)" % (repr(self.addrs))
-
-
-class msg_inv():
-    command = b"inv"
-
-    def __init__(self, inv=None):
-        if inv is None:
-            self.inv = []
-        else:
-            self.inv = inv
-
-    def deserialize(self, f):
-        self.inv = deser_vector(f, CInv)
-
-    def serialize(self):
-        return ser_vector(self.inv)
-
-    def __repr__(self):
-        return "msg_inv(inv=%s)" % (repr(self.inv))
-
-
-class msg_getdata():
-    command = b"getdata"
-
-    def __init__(self, inv=None):
-        self.inv = inv if inv != None else []
-
-    def deserialize(self, f):
-        self.inv = deser_vector(f, CInv)
-
-    def serialize(self):
-        return ser_vector(self.inv)
-
-    def __repr__(self):
-        return "msg_getdata(inv=%s)" % (repr(self.inv))
-
-
-class msg_getblocks():
-    command = b"getblocks"
-
-    def __init__(self):
-        self.locator = CBlockLocator()
-        self.hashstop = 0
-
-    def deserialize(self, f):
-        self.locator = CBlockLocator()
-        self.locator.deserialize(f)
-        self.hashstop = deser_uint256(f)
-
-    def serialize(self):
-        r = b""
-        r += self.locator.serialize()
-        r += ser_uint256(self.hashstop)
-        return r
-
-    def __repr__(self):
-        return "msg_getblocks(locator=%s hashstop=%064x)" \
-            % (repr(self.locator), self.hashstop)
-
-
-class msg_tx():
-    command = b"tx"
-
-    def __init__(self, tx=CTransaction()):
-        self.tx = tx
-
-    def deserialize(self, f):
-        self.tx.deserialize(f)
-
-    def serialize(self):
-        return self.tx.serialize_without_witness()
-
-    def __repr__(self):
-        return "msg_tx(tx=%s)" % (repr(self.tx))
-
-class msg_witness_tx(msg_tx):
-
-    def serialize(self):
-        return self.tx.serialize_with_witness()
-
-
-class msg_block():
-    command = b"block"
-
-    def __init__(self, block=None):
-        if block is None:
-            self.block = CBlock()
-        else:
-            self.block = block
-
-    def deserialize(self, f):
-        self.block.deserialize(f)
-
-    def serialize(self):
-        return self.block.serialize()
-
-    def __repr__(self):
-        return "msg_block(block=%s)" % (repr(self.block))
-
-# for cases where a user needs tighter control over what is sent over the wire
-# note that the user must supply the name of the command, and the data
-class msg_generic():
-    def __init__(self, command, data=None):
-        self.command = command
-        self.data = data
-
-    def serialize(self):
-        return self.data
-
-    def __repr__(self):
-        return "msg_generic()"
-
-class msg_witness_block(msg_block):
-
-    def serialize(self):
-        r = self.block.serialize(with_witness=True)
-        return r
-
-class msg_getaddr():
-    command = b"getaddr"
-
-    def __init__(self):
-        pass
-
-    def deserialize(self, f):
-        pass
-
-    def serialize(self):
-        return b""
-
-    def __repr__(self):
-        return "msg_getaddr()"
-
-
-class msg_ping():
-    command = b"ping"
-
-    def __init__(self, nonce=0):
-        self.nonce = nonce
-
-    def deserialize(self, f):
-        self.nonce = struct.unpack("<Q", f.read(8))[0]
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<Q", self.nonce)
-        return r
-
-    def __repr__(self):
-        return "msg_ping(nonce=%08x)" % self.nonce
-
-
-class msg_pong():
-    command = b"pong"
-
-    def __init__(self, nonce=0):
-        self.nonce = nonce
-
-    def deserialize(self, f):
-        self.nonce = struct.unpack("<Q", f.read(8))[0]
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<Q", self.nonce)
-        return r
-
-    def __repr__(self):
-        return "msg_pong(nonce=%08x)" % self.nonce
-
-
-class msg_mempool():
-    command = b"mempool"
-
-    def __init__(self):
-        pass
-
-    def deserialize(self, f):
-        pass
-
-    def serialize(self):
-        return b""
-
-    def __repr__(self):
-        return "msg_mempool()"
-
-class msg_sendheaders():
-    command = b"sendheaders"
-
-    def __init__(self):
-        pass
-
-    def deserialize(self, f):
-        pass
-
-    def serialize(self):
-        return b""
-
-    def __repr__(self):
-        return "msg_sendheaders()"
-
-
-# getheaders message has
-# number of entries
-# vector of hashes
-# hash_stop (hash of last desired block header, 0 to get as many as possible)
-class msg_getheaders():
-    command = b"getheaders"
-
-    def __init__(self):
-        self.locator = CBlockLocator()
-        self.hashstop = 0
-
-    def deserialize(self, f):
-        self.locator = CBlockLocator()
-        self.locator.deserialize(f)
-        self.hashstop = deser_uint256(f)
-
-    def serialize(self):
-        r = b""
-        r += self.locator.serialize()
-        r += ser_uint256(self.hashstop)
-        return r
-
-    def __repr__(self):
-        return "msg_getheaders(locator=%s, stop=%064x)" \
-            % (repr(self.locator), self.hashstop)
-
-
-# headers message has
-# <count> <vector of block headers>
-class msg_headers():
-    command = b"headers"
-
-    def __init__(self, headers=None):
-        self.headers = headers if headers is not None else []
-
-    def deserialize(self, f):
-        # comment in bitcoind indicates these should be deserialized as blocks
-        blocks = deser_vector(f, CBlock)
-        for x in blocks:
-            self.headers.append(CBlockHeader(x))
-
-    def serialize(self):
-        blocks = [CBlock(x) for x in self.headers]
-        return ser_vector(blocks)
-
-    def __repr__(self):
-        return "msg_headers(headers=%s)" % repr(self.headers)
-
-
-class msg_reject():
-    command = b"reject"
-    REJECT_MALFORMED = 1
-
-    def __init__(self):
-        self.message = b""
-        self.code = 0
-        self.reason = b""
-        self.data = 0
-
-    def deserialize(self, f):
-        self.message = deser_string(f)
-        self.code = struct.unpack("<B", f.read(1))[0]
-        self.reason = deser_string(f)
-        if (self.code != self.REJECT_MALFORMED and
-                (self.message == b"block" or self.message == b"tx")):
-            self.data = deser_uint256(f)
-
-    def serialize(self):
-        r = ser_string(self.message)
-        r += struct.pack("<B", self.code)
-        r += ser_string(self.reason)
-        if (self.code != self.REJECT_MALFORMED and
-                (self.message == b"block" or self.message == b"tx")):
-            r += ser_uint256(self.data)
-        return r
-
-    def __repr__(self):
-        return "msg_reject: %s %d %s [%064x]" \
-            % (self.message, self.code, self.reason, self.data)
-
-class msg_feefilter():
-    command = b"feefilter"
-
-    def __init__(self, feerate=0):
-        self.feerate = feerate
-
-    def deserialize(self, f):
-        self.feerate = struct.unpack("<Q", f.read(8))[0]
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<Q", self.feerate)
-        return r
-
-    def __repr__(self):
-        return "msg_feefilter(feerate=%08x)" % self.feerate
-
-class msg_sendcmpct():
-    command = b"sendcmpct"
-
-    def __init__(self):
-        self.announce = False
-        self.version = 1
-
-    def deserialize(self, f):
-        self.announce = struct.unpack("<?", f.read(1))[0]
-        self.version = struct.unpack("<Q", f.read(8))[0]
-
-    def serialize(self):
-        r = b""
-        r += struct.pack("<?", self.announce)
-        r += struct.pack("<Q", self.version)
-        return r
-
-    def __repr__(self):
-        return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
-
-class msg_cmpctblock():
-    command = b"cmpctblock"
-
-    def __init__(self, header_and_shortids = None):
-        self.header_and_shortids = header_and_shortids
-
-    def deserialize(self, f):
-        self.header_and_shortids = P2PHeaderAndShortIDs()
-        self.header_and_shortids.deserialize(f)
-
-    def serialize(self):
-        r = b""
-        r += self.header_and_shortids.serialize()
-        return r
-
-    def __repr__(self):
-        return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
-
-class msg_getblocktxn():
-    command = b"getblocktxn"
-
-    def __init__(self):
-        self.block_txn_request = None
-
-    def deserialize(self, f):
-        self.block_txn_request = BlockTransactionsRequest()
-        self.block_txn_request.deserialize(f)
-
-    def serialize(self):
-        r = b""
-        r += self.block_txn_request.serialize()
-        return r
-
-    def __repr__(self):
-        return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
-
-class msg_blocktxn():
-    command = b"blocktxn"
-
-    def __init__(self):
-        self.block_transactions = BlockTransactions()
-
-    def deserialize(self, f):
-        self.block_transactions.deserialize(f)
-
-    def serialize(self):
-        r = b""
-        r += self.block_transactions.serialize()
-        return r
-
-    def __repr__(self):
-        return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
-
-class msg_witness_blocktxn(msg_blocktxn):
-    def serialize(self):
-        r = b""
-        r += self.block_transactions.serialize(with_witness=True)
-        return r
-
 class NodeConnCB():
     """Callback and helper functions for P2P connection to a bitcoind node.
 
-- 
cgit v1.2.3