Merge #8916: 0.13.1 backports

9777fe1 remove redundant tests in p2p-segwit.py (Johnson Lau)
fef7b46 test segwit uncompressed key fixes (Johnson Lau)
4ec21e8 Fix ismine and addwitnessaddress: no uncompressed keys in segwit (Pieter Wuille)
908fced [qa] Add tests for uncompressed pubkeys in segwit (Suhas Daftuar)
b4b8527 Make test framework produce lowS signatures (Johnson Lau)
821f3e6 Require compressed keys in segwit as policy and disable signing with uncompressed keys for segwit scripts (Johnson Lau)
540413d Add standard limits for P2WSH with tests (Johnson Lau)
9bb2a02 [qa] Build v4 blocks in p2p-compactblocktests (Matt Corallo)
df5069b [qa] Send segwit-encoded blocktxn messages in p2p-compactblocks (Matt Corallo)
bcf3806 Update bitcoin-tx to output witness data. (jonnynewbs)
cc6f551 [qa] Fix compact block shortids for a test case (Dagur Valberg Johannsson)
4bb9ce8 Use cmpctblock type 2 for segwit-enabled transfer (Matt Corallo)
890ac25 Fix overly-prescriptive p2p-segwit test for new fetch logic (Matt Corallo)
fe1975a Use cmpctblock type 2 for segwit-enabled transfer (Pieter Wuille)
611cc50 [qa] Fix bug in mininode witness deserialization (Suhas Daftuar)
61e282b [qa] Add support for compactblocks v2 to mininode (Suhas Daftuar)
e47299a [qa] Update p2p-compactblocks.py for compactblocks v2 (Suhas Daftuar)
7a34a46 Add NULLDUMMY verify flag in bitcoinconsensus.h (Johnson Lau)
7ae6242 net: fix a few cases where messages were sent rather than dropped upon disconnection (Cory Fields)
3e80ab7 Add policy: null signature for failed CHECK(MULTI)SIG (Johnson Lau)
0027672 Make non-minimal OP_IF/NOTIF argument non-standard for P2WSH (Johnson Lau)

3 files changed, 131 insertions, 9 deletions

diff --git a/qa/rpc-tests/test_framework/address.py b/qa/rpc-tests/test_framework/address.py
new file mode 100644
index 0000000000..50b999be61
--- /dev/null
+++ b/qa/rpc-tests/test_framework/address.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# address.py
+#
+# This file encodes and decodes BASE58 P2PKH and P2SH addresses
+#
+
+from .script import hash256, hash160, sha256, CScript, OP_0
+from .util import bytes_to_hex_str, hex_str_to_bytes
+
+chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+def byte_to_base58(b, version):
+    result = ''
+    str = bytes_to_hex_str(b)
+    str = bytes_to_hex_str(chr(version).encode('latin-1')) + str
+    checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str)))
+    str += checksum[:8]
+    value = int('0x'+str,0)
+    while value > 0:
+        result = chars[value % 58] + result
+        value //= 58
+    while (str[:2] == '00'):
+        result = chars[0] + result
+        str = str[2:]
+    return result
+
+# TODO: def base58_decode
+
+def keyhash_to_p2pkh(hash, main = False):
+    assert (len(hash) == 20)
+    version = 0 if main else 111
+    return byte_to_base58(hash, version)
+
+def scripthash_to_p2sh(hash, main = False):
+    assert (len(hash) == 20)
+    version = 5 if main else 196
+    return byte_to_base58(hash, version)
+
+def key_to_p2pkh(key, main = False):
+    key = check_key(key)
+    return keyhash_to_p2pkh(hash160(key), main)
+
+def script_to_p2sh(script, main = False):
+    script = check_script(script)
+    return scripthash_to_p2sh(hash160(script), main)
+
+def key_to_p2sh_p2wpkh(key, main = False):
+    key = check_key(key)
+    p2shscript = CScript([OP_0, hash160(key)])
+    return script_to_p2sh(p2shscript, main)
+
+def script_to_p2sh_p2wsh(script, main = False):
+    script = check_script(script)
+    p2shscript = CScript([OP_0, sha256(script)])
+    return script_to_p2sh(p2shscript, main)
+
+def check_key(key):
+    if (type(key) is str):
+        key = hex_str_to_bytes(key) # Assuming this is hex string
+    if (type(key) is bytes and (len(key) == 33 or len(key) == 65)):
+        return key
+    assert(False)
+
+def check_script(script):
+    if (type(script) is str):
+        script = hex_str_to_bytes(script) # Assuming this is hex string
+    if (type(script) is bytes or type(script) is CScript):
+        return script
+    assert(False)
diff --git a/qa/rpc-tests/test_framework/key.py b/qa/rpc-tests/test_framework/key.py
index ba3038fe04..c63a15c1e0 100644
--- a/qa/rpc-tests/test_framework/key.py
+++ b/qa/rpc-tests/test_framework/key.py
@@ -75,6 +75,9 @@ ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p,
 # this specifies the curve used with ECDSA.
 NID_secp256k1 = 714 # from openssl/obj_mac.h
 
+SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141
+SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2
+
 # Thx to Sam Devlin for the ctypes magic 64-bit fix.
 def _check_result(val, func, args):
     if val == 0:
@@ -147,7 +150,7 @@ class CECKey(object):
         r = self.get_raw_ecdh_key(other_pubkey)
         return kdf(r)
 
-    def sign(self, hash):
+    def sign(self, hash, low_s = True):
         # FIXME: need unit tests for below cases
         if not isinstance(hash, bytes):
             raise TypeError('Hash must be bytes instance; got %r' % hash.__class__)
@@ -159,7 +162,25 @@ class CECKey(object):
         mb_sig = ctypes.create_string_buffer(sig_size0.value)
         result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k)
         assert 1 == result
-        return mb_sig.raw[:sig_size0.value]
+        assert mb_sig.raw[0] == 0x30
+        assert mb_sig.raw[1] == sig_size0.value - 2
+        total_size = mb_sig.raw[1]
+        assert mb_sig.raw[2] == 2
+        r_size = mb_sig.raw[3]
+        assert mb_sig.raw[4 + r_size] == 2
+        s_size = mb_sig.raw[5 + r_size]
+        s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big')
+        if (not low_s) or s_value <= SECP256K1_ORDER_HALF:
+            return mb_sig.raw[:sig_size0.value]
+        else:
+            low_s_value = SECP256K1_ORDER - s_value
+            low_s_bytes = (low_s_value).to_bytes(33, byteorder='big')
+            while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80:
+                low_s_bytes = low_s_bytes[1:]
+            new_s_size = len(low_s_bytes)
+            new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big')
+            new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big')
+            return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes
 
     def verify(self, hash, sig):
         """Verify a DER signature"""
diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index 88a3b7e0f7..4d238c08d9 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -452,7 +452,7 @@ class CTransaction(object):
         else:
             self.vout = deser_vector(f, CTxOut)
         if flags != 0:
-            self.wit.vtxinwit = [CTxInWitness()]*len(self.vin)
+            self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))]
             self.wit.deserialize(f)
         self.nLockTime = struct.unpack("<I", f.read(4))[0]
         self.sha256 = None
@@ -518,8 +518,8 @@ class CTransaction(object):
         return True
 
     def __repr__(self):
-        return "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i)" \
-            % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime)
+        return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \
+            % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime)
 
 
 class CBlockHeader(object):
@@ -755,6 +755,9 @@ class PrefilledTransaction(object):
             r += self.tx.serialize_without_witness()
         return r
 
+    def serialize_with_witness(self):
+        return self.serialize(with_witness=True)
+
     def __repr__(self):
         return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
 
@@ -779,6 +782,7 @@ class P2PHeaderAndShortIDs(object):
         self.prefilled_txn = deser_vector(f, PrefilledTransaction)
         self.prefilled_txn_length = len(self.prefilled_txn)
 
+    # When using version 2 compact blocks, we must serialize with_witness.
     def serialize(self, with_witness=False):
         r = b""
         r += self.header.serialize()
@@ -787,12 +791,20 @@ class P2PHeaderAndShortIDs(object):
         for x in self.shortids:
             # We only want the first 6 bytes
             r += struct.pack("<Q", x)[0:6]
-        r += ser_vector(self.prefilled_txn)
+        if with_witness:
+            r += ser_vector(self.prefilled_txn, "serialize_with_witness")
+        else:
+            r += ser_vector(self.prefilled_txn)
         return r
 
     def __repr__(self):
         return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
 
+# P2P version of the above that will use witness serialization (for compact
+# block version 2)
+class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs):
+    def serialize(self):
+        return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True)
 
 # Calculate the BIP 152-compact blocks shortid for a given transaction hash
 def calculate_shortid(k0, k1, tx_hash):
@@ -808,6 +820,7 @@ class HeaderAndShortIDs(object):
         self.nonce = 0
         self.shortids = []
         self.prefilled_txn = []
+        self.use_witness = False
 
         if p2pheaders_and_shortids != None:
             self.header = p2pheaders_and_shortids.header
@@ -819,7 +832,10 @@ class HeaderAndShortIDs(object):
                 last_index = self.prefilled_txn[-1].index
 
     def to_p2p(self):
-        ret = P2PHeaderAndShortIDs()
+        if self.use_witness:
+            ret = P2PHeaderAndShortWitnessIDs()
+        else:
+            ret = P2PHeaderAndShortIDs()
         ret.header = self.header
         ret.nonce = self.nonce
         ret.shortids_length = len(self.shortids)
@@ -840,15 +856,20 @@ class HeaderAndShortIDs(object):
         key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
         return [ key0, key1 ]
 
-    def initialize_from_block(self, block, nonce=0, prefill_list = [0]):
+    # Version 2 compact blocks use wtxid in shortids (rather than txid)
+    def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False):
         self.header = CBlockHeader(block)
         self.nonce = nonce
         self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
         self.shortids = []
+        self.use_witness = use_witness
         [k0, k1] = self.get_siphash_keys()
         for i in range(len(block.vtx)):
             if i not in prefill_list:
-                self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256))
+                tx_hash = block.vtx[i].sha256
+                if use_witness:
+                    tx_hash = block.vtx[i].calc_sha256(with_witness=True)
+                self.shortids.append(calculate_shortid(k0, k1, tx_hash))
 
     def __repr__(self):
         return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
@@ -1424,6 +1445,12 @@ class msg_blocktxn(object):
     def __repr__(self):
         return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
 
+class msg_witness_blocktxn(msg_blocktxn):
+    def serialize(self):
+        r = b""
+        r += self.block_transactions.serialize(with_witness=True)
+        return r
+
 # This is what a callback should look like for NodeConn
 # Reimplement the on_* functions to provide handling for events
 class NodeConnCB(object):