Merge #10190: [tests] mining functional tests (including regression test for submitblock)

11ba8e9 [tests] rename getblocktemplate_proposals.py to mining.py (John Newbery)
b29dd41 [tests] add test for submit block (John Newbery)
9bf0d80 [tests] run successful test in getblocktemplate first (John Newbery)
82dc597 [tests] don't build blocks manually in getblocktemplate test (John Newbery)
f82c709 [tests] clarify assertions in getblocktemplate test (John Newbery)
66c570a [tests] Don't build the coinbase manually in getblocktemplate test (John Newbery)
38b38cd [tests] getblocktemplate_proposals.py: add logging (John Newbery)
0a3a5ff [tests] Fix flake8 warnings in getblocktemplate tests (John Newbery)
32cffe6 [tests] Fix import order in getblocktemplate test (John Newbery)

Tree-SHA512: a51a57314fa1c4c4b8a7896492ec6e677b6bed12387060def34a62e9dfbee7961f71bb5553fbd70028be61ae32eccf13fd255fa9651f908e9a5e64c28f43f00e

3 files changed, 125 insertions, 158 deletions

diff --git a/test/functional/getblocktemplate_proposals.py b/test/functional/getblocktemplate_proposals.py
deleted file mode 100755
index fca99c7df5..0000000000
--- a/test/functional/getblocktemplate_proposals.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2014-2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test block proposals with getblocktemplate."""
-
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import *
-
-from binascii import a2b_hex, b2a_hex
-from hashlib import sha256
-from struct import pack
-
-def b2x(b):
-    return b2a_hex(b).decode('ascii')
-
-# NOTE: This does not work for signed numbers (set the high bit) or zero (use b'\0')
-def encodeUNum(n):
-    s = bytearray(b'\1')
-    while n > 127:
-        s[0] += 1
-        s.append(n % 256)
-        n //= 256
-    s.append(n)
-    return bytes(s)
-
-def varlenEncode(n):
-    if n < 0xfd:
-        return pack('<B', n)
-    if n <= 0xffff:
-        return b'\xfd' + pack('<H', n)
-    if n <= 0xffffffff:
-        return b'\xfe' + pack('<L', n)
-    return b'\xff' + pack('<Q', n)
-
-def dblsha(b):
-    return sha256(sha256(b).digest()).digest()
-
-def genmrklroot(leaflist):
-    cur = leaflist
-    while len(cur) > 1:
-        n = []
-        if len(cur) & 1:
-            cur.append(cur[-1])
-        for i in range(0, len(cur), 2):
-            n.append(dblsha(cur[i] + cur[i+1]))
-        cur = n
-    return cur[0]
-
-def template_to_bytearray(tmpl, txlist):
-    blkver = pack('<L', tmpl['version'])
-    mrklroot = genmrklroot(list(dblsha(a) for a in txlist))
-    timestamp = pack('<L', tmpl['curtime'])
-    nonce = b'\0\0\0\0'
-    blk = blkver + a2b_hex(tmpl['previousblockhash'])[::-1] + mrklroot + timestamp + a2b_hex(tmpl['bits'])[::-1] + nonce
-    blk += varlenEncode(len(txlist))
-    for tx in txlist:
-        blk += tx
-    return bytearray(blk)
-
-def template_to_hex(tmpl, txlist):
-    return b2x(template_to_bytearray(tmpl, txlist))
-
-def assert_template(node, tmpl, txlist, expect):
-    rsp = node.getblocktemplate({'data':template_to_hex(tmpl, txlist),'mode':'proposal'})
-    if rsp != expect:
-        raise AssertionError('unexpected: %s' % (rsp,))
-
-class GetBlockTemplateProposalTest(BitcoinTestFramework):
-
-    def __init__(self):
-        super().__init__()
-        self.num_nodes = 2
-        self.setup_clean_chain = False
-
-    def run_test(self):
-        node = self.nodes[0]
-        node.generate(1) # Mine a block to leave initial block download
-        tmpl = node.getblocktemplate()
-        if 'coinbasetxn' not in tmpl:
-            rawcoinbase = encodeUNum(tmpl['height'])
-            rawcoinbase += b'\x01-'
-            hexcoinbase = b2x(rawcoinbase)
-            hexoutval = b2x(pack('<Q', tmpl['coinbasevalue']))
-            tmpl['coinbasetxn'] = {'data': '01000000' + '01' + '0000000000000000000000000000000000000000000000000000000000000000ffffffff' + ('%02x' % (len(rawcoinbase),)) + hexcoinbase + 'fffffffe' + '01' + hexoutval + '00' + '00000000'}
-        txlist = list(bytearray(a2b_hex(a['data'])) for a in (tmpl['coinbasetxn'],) + tuple(tmpl['transactions']))
-
-        # Test 0: Capability advertised
-        assert('proposal' in tmpl['capabilities'])
-
-        # NOTE: This test currently FAILS (regtest mode doesn't enforce block height in coinbase)
-        ## Test 1: Bad height in coinbase
-        #txlist[0][4+1+36+1+1] += 1
-        #assert_template(node, tmpl, txlist, 'FIXME')
-        #txlist[0][4+1+36+1+1] -= 1
-
-        # Test 2: Bad input hash for gen tx
-        txlist[0][4+1] += 1
-        assert_template(node, tmpl, txlist, 'bad-cb-missing')
-        txlist[0][4+1] -= 1
-
-        # Test 3: Truncated final tx
-        lastbyte = txlist[-1].pop()
-        assert_raises_jsonrpc(-22, "Block decode failed", assert_template, node, tmpl, txlist, 'n/a')
-        txlist[-1].append(lastbyte)
-
-        # Test 4: Add an invalid tx to the end (duplicate of gen tx)
-        txlist.append(txlist[0])
-        assert_template(node, tmpl, txlist, 'bad-txns-duplicate')
-        txlist.pop()
-
-        # Test 5: Add an invalid tx to the end (non-duplicate)
-        txlist.append(bytearray(txlist[0]))
-        txlist[-1][4+1] = 0xff
-        assert_template(node, tmpl, txlist, 'bad-txns-inputs-missingorspent')
-        txlist.pop()
-
-        # Test 6: Future tx lock time
-        txlist[0][-4:] = b'\xff\xff\xff\xff'
-        assert_template(node, tmpl, txlist, 'bad-txns-nonfinal')
-        txlist[0][-4:] = b'\0\0\0\0'
-
-        # Test 7: Bad tx count
-        txlist.append(b'')
-        assert_raises_jsonrpc(-22, 'Block decode failed', assert_template, node, tmpl, txlist, 'n/a')
-        txlist.pop()
-
-        # Test 8: Bad bits
-        realbits = tmpl['bits']
-        tmpl['bits'] = '1c0000ff'  # impossible in the real world
-        assert_template(node, tmpl, txlist, 'bad-diffbits')
-        tmpl['bits'] = realbits
-
-        # Test 9: Bad merkle root
-        rawtmpl = template_to_bytearray(tmpl, txlist)
-        rawtmpl[4+32] = (rawtmpl[4+32] + 1) % 0x100
-        rsp = node.getblocktemplate({'data':b2x(rawtmpl),'mode':'proposal'})
-        if rsp != 'bad-txnmrklroot':
-            raise AssertionError('unexpected: %s' % (rsp,))
-
-        # Test 10: Bad timestamps
-        realtime = tmpl['curtime']
-        tmpl['curtime'] = 0x7fffffff
-        assert_template(node, tmpl, txlist, 'time-too-new')
-        tmpl['curtime'] = 0
-        assert_template(node, tmpl, txlist, 'time-too-old')
-        tmpl['curtime'] = realtime
-
-        # Test 11: Valid block
-        assert_template(node, tmpl, txlist, None)
-
-        # Test 12: Orphan block
-        tmpl['previousblockhash'] = 'ff00' * 16
-        assert_template(node, tmpl, txlist, 'inconclusive-not-best-prevblk')
-
-if __name__ == '__main__':
-    GetBlockTemplateProposalTest().main()
diff --git a/test/functional/mining.py b/test/functional/mining.py
new file mode 100755
index 0000000000..dbd4e29eca
--- /dev/null
+++ b/test/functional/mining.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test mining RPCs
+
+- getblocktemplate proposal mode
+- submitblock"""
+
+from binascii import b2a_hex
+import copy
+
+from test_framework.blocktools import create_coinbase
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.mininode import CBlock
+from test_framework.util import *
+
+def b2x(b):
+    return b2a_hex(b).decode('ascii')
+
+def assert_template(node, block, expect, rehash=True):
+    if rehash:
+        block.hashMerkleRoot = block.calc_merkle_root()
+    rsp = node.getblocktemplate({'data': b2x(block.serialize()), 'mode': 'proposal'})
+    assert_equal(rsp, expect)
+
+class MiningTest(BitcoinTestFramework):
+
+    def __init__(self):
+        super().__init__()
+        self.num_nodes = 2
+        self.setup_clean_chain = False
+
+    def run_test(self):
+        node = self.nodes[0]
+        # Mine a block to leave initial block download
+        node.generate(1)
+        tmpl = node.getblocktemplate()
+        self.log.info("getblocktemplate: Test capability advertised")
+        assert 'proposal' in tmpl['capabilities']
+        assert 'coinbasetxn' not in tmpl
+
+        coinbase_tx = create_coinbase(height=int(tmpl["height"]) + 1)
+        # sequence numbers must not be max for nLockTime to have effect
+        coinbase_tx.vin[0].nSequence = 2 ** 32 - 2
+        coinbase_tx.rehash()
+
+        block = CBlock()
+        block.nVersion = tmpl["version"]
+        block.hashPrevBlock = int(tmpl["previousblockhash"], 16)
+        block.nTime = tmpl["curtime"]
+        block.nBits = int(tmpl["bits"], 16)
+        block.nNonce = 0
+        block.vtx = [coinbase_tx]
+
+        self.log.info("getblocktemplate: Test valid block")
+        assert_template(node, block, None)
+
+        self.log.info("submitblock: Test block decode failure")
+        assert_raises_jsonrpc(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15]))
+
+        self.log.info("getblocktemplate: Test bad input hash for coinbase transaction")
+        bad_block = copy.deepcopy(block)
+        bad_block.vtx[0].vin[0].prevout.hash += 1
+        bad_block.vtx[0].rehash()
+        assert_template(node, bad_block, 'bad-cb-missing')
+
+        self.log.info("submitblock: Test invalid coinbase transaction")
+        assert_raises_jsonrpc(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize()))
+
+        self.log.info("getblocktemplate: Test truncated final transaction")
+        assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal'})
+
+        self.log.info("getblocktemplate: Test duplicate transaction")
+        bad_block = copy.deepcopy(block)
+        bad_block.vtx.append(bad_block.vtx[0])
+        assert_template(node, bad_block, 'bad-txns-duplicate')
+
+        self.log.info("getblocktemplate: Test invalid transaction")
+        bad_block = copy.deepcopy(block)
+        bad_tx = copy.deepcopy(bad_block.vtx[0])
+        bad_tx.vin[0].prevout.hash = 255
+        bad_tx.rehash()
+        bad_block.vtx.append(bad_tx)
+        assert_template(node, bad_block, 'bad-txns-inputs-missingorspent')
+
+        self.log.info("getblocktemplate: Test nonfinal transaction")
+        bad_block = copy.deepcopy(block)
+        bad_block.vtx[0].nLockTime = 2 ** 32 - 1
+        bad_block.vtx[0].rehash()
+        assert_template(node, bad_block, 'bad-txns-nonfinal')
+
+        self.log.info("getblocktemplate: Test bad tx count")
+        # The tx count is immediately after the block header
+        TX_COUNT_OFFSET = 80
+        bad_block_sn = bytearray(block.serialize())
+        assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1)
+        bad_block_sn[TX_COUNT_OFFSET] += 1
+        assert_raises_jsonrpc(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal'})
+
+        self.log.info("getblocktemplate: Test bad bits")
+        bad_block = copy.deepcopy(block)
+        bad_block.nBits = 469762303  # impossible in the real world
+        assert_template(node, bad_block, 'bad-diffbits')
+
+        self.log.info("getblocktemplate: Test bad merkle root")
+        bad_block = copy.deepcopy(block)
+        bad_block.hashMerkleRoot += 1
+        assert_template(node, bad_block, 'bad-txnmrklroot', False)
+
+        self.log.info("getblocktemplate: Test bad timestamps")
+        bad_block = copy.deepcopy(block)
+        bad_block.nTime = 2 ** 31 - 1
+        assert_template(node, bad_block, 'time-too-new')
+        bad_block.nTime = 0
+        assert_template(node, bad_block, 'time-too-old')
+
+        self.log.info("getblocktemplate: Test not best block")
+        bad_block = copy.deepcopy(block)
+        bad_block.hashPrevBlock = 123
+        assert_template(node, bad_block, 'inconclusive-not-best-prevblk')
+
+if __name__ == '__main__':
+    MiningTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 54f625514b..b7bc6e841b 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -108,6 +108,7 @@ BASE_SCRIPTS= [
     'signmessages.py',
     'nulldummy.py',
     'import-rescan.py',
+    'mining.py',
     'bumpfee.py',
     'rpcnamedargs.py',
     'listsinceblock.py',
@@ -141,7 +142,6 @@ EXTENDED_SCRIPTS = [
     'bipdersig-p2p.py',
     'bipdersig.py',
     'example_test.py',
-    'getblocktemplate_proposals.py',
     'txn_doublespend.py',
     'txn_clone.py --mineblock',
     'forknotify.py',