diff options
Diffstat (limited to 'qa/rpc-tests/test_framework')
| -rwxr-xr-x | qa/rpc-tests/test_framework/mininode.py | 373 | ||||
| -rw-r--r-- | qa/rpc-tests/test_framework/siphash.py | 64 |
2 files changed, 361 insertions, 76 deletions
diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index cdd5292cd6..caffab3535 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,9 +36,10 @@ from threading import RLock from threading import Thread import logging import copy +from test_framework.siphash import siphash256 BIP0031_VERSION = 60000 -MY_VERSION = 60001 # past bip-31 for ping/pong +MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MAX_INV_SZ = 50000 @@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -74,8 +75,19 @@ def ripemd160(s): def hash256(s): return sha256(sha256(s)) +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r -def deser_string(f): +def deser_compact_size(f): nit = struct.unpack("<B", f.read(1))[0] if nit == 253: nit = struct.unpack("<H", f.read(2))[0] @@ -83,16 +95,14 @@ def deser_string(f): nit = struct.unpack("<I", f.read(4))[0] elif nit == 255: nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) return f.read(nit) def ser_string(s): - if len(s) < 253: - return struct.pack("B", len(s)) + s - elif len(s) < 0x10000: - return struct.pack("<BH", 253, len(s)) + s - elif len(s) < 0x100000000: - return struct.pack("<BI", 254, len(s)) + s - return struct.pack("<BQ", 255, len(s)) + s + return ser_compact_size(len(s)) + s def deser_uint256(f): r = 0 @@ -125,13 +135,7 @@ def uint256_from_compact(c): def deser_vector(f, c): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = c() @@ -144,15 +148,7 @@ def deser_vector(f, c): # entries in the vector (we use this for serializing the vector of transactions # for a witness block). def ser_vector(l, ser_function_name=None): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: if ser_function_name: r += getattr(i, ser_function_name)() @@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None): def deser_uint256_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_uint256(f) @@ -177,28 +167,14 @@ def deser_uint256_vector(f): def ser_uint256_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += ser_uint256(i) return r def deser_string_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_string(f) @@ -207,28 +183,14 @@ def deser_string_vector(f): def ser_string_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for sv in l: r += ser_string(sv) return r def deser_int_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = struct.unpack("<i", f.read(4))[0] @@ -237,15 +199,7 @@ def deser_int_vector(f): def ser_int_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += struct.pack("<i", i) return r @@ -294,7 +248,8 @@ class CInv(object): 1: "TX", 2: "Block", 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock" + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" } def __init__(self, t=0, h=0): @@ -781,6 +736,187 @@ class CAlert(object): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + r += ser_vector(self.prefilled_txn) + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(object): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + def initialize_from_block(self, block, nonce=0, prefill_list = [0]): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(object): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(object): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions) + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + # Objects that correspond to messages on the wire class msg_version(object): command = b"version" @@ -1215,6 +1351,79 @@ class msg_feefilter(object): def __repr__(self): return "msg_feefilter(feerate=%08x)" % self.feerate +class msg_sendcmpct(object): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(object): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(object): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(object): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize() + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + # This is what a callback should look like for NodeConn # Reimplement the on_* functions to provide handling for events class NodeConnCB(object): @@ -1295,6 +1504,10 @@ class NodeConnCB(object): def on_pong(self, conn, message): pass def on_feefilter(self, conn, message): pass def on_sendheaders(self, conn, message): pass + def on_sendcmpct(self, conn, message): pass + def on_cmpctblock(self, conn, message): pass + def on_getblocktxn(self, conn, message): pass + def on_blocktxn(self, conn, message): pass # More useful callbacks and functions for NodeConnCB's which have a single NodeConn class SingleNodeConnCB(NodeConnCB): @@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB): def send_message(self, message): self.connection.send_message(message) + def send_and_ping(self, message): + self.send_message(message) + self.sync_with_ping() + def on_pong(self, conn, message): self.last_pong = message @@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher): b"reject": msg_reject, b"mempool": msg_mempool, b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders + b"sendheaders": msg_sendheaders, + b"sendcmpct": msg_sendcmpct, + b"cmpctblock": msg_cmpctblock, + b"getblocktxn": msg_getblocktxn, + b"blocktxn": msg_blocktxn } MAGIC_BYTES = { "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py new file mode 100644 index 0000000000..9c0574bd93 --- /dev/null +++ b/qa/rpc-tests/test_framework/siphash.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# siphash.py - Specialized SipHash-2-4 implementations +# +# This implements SipHash-2-4 for 256-bit integers. + +def rotl64(n, b): + return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + +def siphash_round(v0, v1, v2, v3): + v0 = (v0 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 13) + v1 ^= v0 + v0 = rotl64(v0, 32) + v2 = (v2 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 16) + v3 ^= v2 + v0 = (v0 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 21) + v3 ^= v0 + v2 = (v2 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 17) + v1 ^= v2 + v2 = rotl64(v2, 32) + return (v0, v1, v2, v3) + +def siphash256(k0, k1, h): + n0 = h & ((1 << 64) - 1) + n1 = (h >> 64) & ((1 << 64) - 1) + n2 = (h >> 128) & ((1 << 64) - 1) + n3 = (h >> 192) & ((1 << 64) - 1) + v0 = 0x736f6d6570736575 ^ k0 + v1 = 0x646f72616e646f6d ^ k1 + v2 = 0x6c7967656e657261 ^ k0 + v3 = 0x7465646279746573 ^ k1 ^ n0 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n0 + v3 ^= n1 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n1 + v3 ^= n2 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n2 + v3 ^= n3 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n3 + v3 ^= 0x2000000000000000 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= 0x2000000000000000 + v2 ^= 0xFF + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + return v0 ^ v1 ^ v2 ^ v3 |