Merge #11818: I accidentally [deliberately] killed it [the ComparisonTestFramework]

9c92c8c827 [tests] Remove Comparison Test Framework (John Newbery)
e80c640d78 [tests] Remove bip9-softforks.py (John Newbery)

Pull request description:

  Builds on #11772, #11773 and #11817. Please review those PRs first.

  Final step in #10603.

  - First commit removes bip9-softforks.py.  bip9-sofforks.py was intended to be a generic test for versionbits deployments. However, it only tests CSV activation and was not updated to test segwit activation. CSV activation is tested by bip68-112-113-p2p.py, so this test is duplicated effort. Rather than try to update it to use the BitcoinTestFramework, just remove it. (see https://github.com/btcdrak/bitcoin/pull/8 for previous discussion around the redundancy of bip9-softforks.py)
  - Second commit removes the now unused BitcoinComparisonFramework class and the comptool and blockstore modules.

Tree-SHA512: 4bb7196d521048b3b8ba95c87dde73005a1ac73d29ccbb869f11ce9a71089686e7eacd7335337853041dfbd3a5b110172b105adbada58779814d4db22b1376f5

3 files changed, 0 insertions, 586 deletions

diff --git a/test/functional/test_framework/blockstore.py b/test/functional/test_framework/blockstore.py
deleted file mode 100644
index 6073285a6c..0000000000
--- a/test/functional/test_framework/blockstore.py
+++ /dev/null
@@ -1,160 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2015-2017 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""BlockStore and TxStore helper classes."""
-
-from .mininode import *
-from io import BytesIO
-import dbm.dumb as dbmd
-
-logger = logging.getLogger("TestFramework.blockstore")
-
-class BlockStore():
-    """BlockStore helper class.
-
-    BlockStore keeps a map of blocks and implements helper functions for
-    responding to getheaders and getdata, and for constructing a getheaders
-    message.
-    """
-
-    def __init__(self, datadir):
-        self.blockDB = dbmd.open(datadir + "/blocks", 'c')
-        self.currentBlock = 0
-        self.headers_map = dict()
-
-    def close(self):
-        self.blockDB.close()
-
-    def erase(self, blockhash):
-        del self.blockDB[repr(blockhash)]
-
-    # lookup an entry and return the item as raw bytes
-    def get(self, blockhash):
-        value = None
-        try:
-            value = self.blockDB[repr(blockhash)]
-        except KeyError:
-            return None
-        return value
-
-    # lookup an entry and return it as a CBlock
-    def get_block(self, blockhash):
-        ret = None
-        serialized_block = self.get(blockhash)
-        if serialized_block is not None:
-            f = BytesIO(serialized_block)
-            ret = CBlock()
-            ret.deserialize(f)
-            ret.calc_sha256()
-        return ret
-
-    def get_header(self, blockhash):
-        try:
-            return self.headers_map[blockhash]
-        except KeyError:
-            return None
-
-    # Note: this pulls full blocks out of the database just to retrieve
-    # the headers -- perhaps we could keep a separate data structure
-    # to avoid this overhead.
-    def headers_for(self, locator, hash_stop, current_tip=None):
-        if current_tip is None:
-            current_tip = self.currentBlock
-        current_block_header = self.get_header(current_tip)
-        if current_block_header is None:
-            return None
-
-        response = msg_headers()
-        headersList = [ current_block_header ]
-        maxheaders = 2000
-        while (headersList[0].sha256 not in locator.vHave):
-            prevBlockHash = headersList[0].hashPrevBlock
-            prevBlockHeader = self.get_header(prevBlockHash)
-            if prevBlockHeader is not None:
-                headersList.insert(0, prevBlockHeader)
-            else:
-                break
-        headersList = headersList[:maxheaders] # truncate if we have too many
-        hashList = [x.sha256 for x in headersList]
-        index = len(headersList)
-        if (hash_stop in hashList):
-            index = hashList.index(hash_stop)+1
-        response.headers = headersList[:index]
-        return response
-
-    def add_block(self, block):
-        block.calc_sha256()
-        try:
-            self.blockDB[repr(block.sha256)] = bytes(block.serialize())
-        except TypeError:
-            logger.exception("Unexpected error")
-        self.currentBlock = block.sha256
-        self.headers_map[block.sha256] = CBlockHeader(block)
-
-    def add_header(self, header):
-        self.headers_map[header.sha256] = header
-
-    # lookup the hashes in "inv", and return p2p messages for delivering
-    # blocks found.
-    def get_blocks(self, inv):
-        responses = []
-        for i in inv:
-            if (i.type == 2 or i.type == (2 | (1 << 30))): # MSG_BLOCK or MSG_WITNESS_BLOCK
-                data = self.get(i.hash)
-                if data is not None:
-                    # Use msg_generic to avoid re-serialization
-                    responses.append(msg_generic(b"block", data))
-        return responses
-
-    def get_locator(self, current_tip=None):
-        if current_tip is None:
-            current_tip = self.currentBlock
-        r = []
-        counter = 0
-        step = 1
-        lastBlock = self.get_block(current_tip)
-        while lastBlock is not None:
-            r.append(lastBlock.hashPrevBlock)
-            for i in range(step):
-                lastBlock = self.get_block(lastBlock.hashPrevBlock)
-                if lastBlock is None:
-                    break
-            counter += 1
-            if counter > 10:
-                step *= 2
-        locator = CBlockLocator()
-        locator.vHave = r
-        return locator
-
-class TxStore():
-    def __init__(self, datadir):
-        self.txDB = dbmd.open(datadir + "/transactions", 'c')
-
-    def close(self):
-        self.txDB.close()
-
-    # lookup an entry and return the item as raw bytes
-    def get(self, txhash):
-        value = None
-        try:
-            value = self.txDB[repr(txhash)]
-        except KeyError:
-            return None
-        return value
-
-    def add_transaction(self, tx):
-        tx.calc_sha256()
-        try:
-            self.txDB[repr(tx.sha256)] = bytes(tx.serialize())
-        except TypeError:
-            logger.exception("Unexpected error")
-
-    def get_transactions(self, inv):
-        responses = []
-        for i in inv:
-            if (i.type == 1 or i.type == (1 | (1 << 30))): # MSG_TX or MSG_WITNESS_TX
-                tx = self.get(i.hash)
-                if tx is not None:
-                    responses.append(msg_generic(b"tx", tx))
-        return responses
diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py
deleted file mode 100755
index e0ca78e5d1..0000000000
--- a/test/functional/test_framework/comptool.py
+++ /dev/null
@@ -1,397 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2015-2017 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Compare two or more bitcoinds to each other.
-
-To use, create a class that implements get_tests(), and pass it in
-as the test generator to TestManager.  get_tests() should be a python
-generator that returns TestInstance objects.  See below for definition.
-
-TestP2PConn behaves as follows:
-    Configure with a BlockStore and TxStore
-    on_inv: log the message but don't request
-    on_headers: log the chain tip
-    on_pong: update ping response map (for synchronization)
-    on_getheaders: provide headers via BlockStore
-    on_getdata: provide blocks via BlockStore
-"""
-
-from .mininode import *
-from .blockstore import BlockStore, TxStore
-from .util import p2p_port, wait_until
-
-import logging
-
-logger=logging.getLogger("TestFramework.comptool")
-
-global mininode_lock
-
-class RejectResult():
-    """Outcome that expects rejection of a transaction or block."""
-    def __init__(self, code, reason=b''):
-        self.code = code
-        self.reason = reason
-    def match(self, other):
-        if self.code != other.code:
-            return False
-        return other.reason.startswith(self.reason)
-    def __repr__(self):
-        return '%i:%s' % (self.code,self.reason or '*')
-
-class TestP2PConn(P2PInterface):
-
-    def __init__(self, block_store, tx_store):
-        super().__init__()
-        self.bestblockhash = None
-        self.block_store = block_store
-        self.block_request_map = {}
-        self.tx_store = tx_store
-        self.tx_request_map = {}
-        self.block_reject_map = {}
-        self.tx_reject_map = {}
-
-        # When the pingmap is non-empty we're waiting for 
-        # a response
-        self.pingMap = {} 
-        self.lastInv = []
-        self.closed = False
-
-    def on_close(self):
-        self.closed = True
-
-    def on_headers(self, message):
-        if len(message.headers) > 0:
-            best_header = message.headers[-1]
-            best_header.calc_sha256()
-            self.bestblockhash = best_header.sha256
-
-    def on_getheaders(self, message):
-        response = self.block_store.headers_for(message.locator, message.hashstop)
-        if response is not None:
-            self.send_message(response)
-
-    def on_getdata(self, message):
-        [self.send_message(r) for r in self.block_store.get_blocks(message.inv)]
-        [self.send_message(r) for r in self.tx_store.get_transactions(message.inv)]
-
-        for i in message.inv:
-            if i.type == 1 or i.type == 1 | (1 << 30): # MSG_TX or MSG_WITNESS_TX
-                self.tx_request_map[i.hash] = True
-            elif i.type == 2 or i.type == 2 | (1 << 30): # MSG_BLOCK or MSG_WITNESS_BLOCK
-                self.block_request_map[i.hash] = True
-
-    def on_inv(self, message):
-        self.lastInv = [x.hash for x in message.inv]
-
-    def on_pong(self, message):
-        try:
-            del self.pingMap[message.nonce]
-        except KeyError:
-            raise AssertionError("Got pong for unknown ping [%s]" % repr(message))
-
-    def on_reject(self, message):
-        if message.message == b'tx':
-            self.tx_reject_map[message.data] = RejectResult(message.code, message.reason)
-        if message.message == b'block':
-            self.block_reject_map[message.data] = RejectResult(message.code, message.reason)
-
-    def send_inv(self, obj):
-        mtype = 2 if isinstance(obj, CBlock) else 1
-        self.send_message(msg_inv([CInv(mtype, obj.sha256)]))
-
-    def send_getheaders(self):
-        # We ask for headers from their last tip.
-        m = msg_getheaders()
-        m.locator = self.block_store.get_locator(self.bestblockhash)
-        self.send_message(m)
-
-    def send_header(self, header):
-        m = msg_headers()
-        m.headers.append(header)
-        self.send_message(m)
-
-    # This assumes BIP31
-    def send_ping(self, nonce):
-        self.pingMap[nonce] = True
-        self.send_message(msg_ping(nonce))
-
-    def received_ping_response(self, nonce):
-        return nonce not in self.pingMap
-
-    def send_mempool(self):
-        self.lastInv = []
-        self.send_message(msg_mempool())
-
-# TestInstance:
-#
-# Instances of these are generated by the test generator, and fed into the
-# comptool.
-#
-# "blocks_and_transactions" should be an array of
-#    [obj, True/False/None, hash/None]:
-#  - obj is either a CBlock, CBlockHeader, or a CTransaction, and
-#  - the second value indicates whether the object should be accepted
-#    into the blockchain or mempool (for tests where we expect a certain
-#    answer), or "None" if we don't expect a certain answer and are just
-#    comparing the behavior of the nodes being tested.
-#  - the third value is the hash to test the tip against (if None or omitted,
-#    use the hash of the block)
-#  - NOTE: if a block header, no test is performed; instead the header is
-#    just added to the block_store.  This is to facilitate block delivery
-#    when communicating with headers-first clients (when withholding an
-#    intermediate block).
-# sync_every_block: if True, then each block will be inv'ed, synced, and
-#    nodes will be tested based on the outcome for the block.  If False,
-#    then inv's accumulate until all blocks are processed (or max inv size
-#    is reached) and then sent out in one inv message.  Then the final block
-#    will be synced across all connections, and the outcome of the final 
-#    block will be tested.
-# sync_every_tx: analogous to behavior for sync_every_block, except if outcome
-#    on the final tx is None, then contents of entire mempool are compared
-#    across all connections.  (If outcome of final tx is specified as true
-#    or false, then only the last tx is tested against outcome.)
-
-class TestInstance():
-    def __init__(self, objects=None, sync_every_block=True, sync_every_tx=False):
-        self.blocks_and_transactions = objects if objects else []
-        self.sync_every_block = sync_every_block
-        self.sync_every_tx = sync_every_tx
-
-class TestManager():
-
-    def __init__(self, testgen, datadir):
-        self.test_generator = testgen
-        self.p2p_connections= []
-        self.block_store    = BlockStore(datadir)
-        self.tx_store       = TxStore(datadir)
-        self.ping_counter   = 1
-
-    def add_all_connections(self, nodes):
-        for i in range(len(nodes)):
-            # Create a p2p connection to each node
-            node = TestP2PConn(self.block_store, self.tx_store)
-            node.peer_connect('127.0.0.1', p2p_port(i))
-            self.p2p_connections.append(node)
-
-    def clear_all_connections(self):
-        self.p2p_connections = []
-
-    def wait_for_disconnections(self):
-        def disconnected():
-            return all(node.closed for node in self.p2p_connections)
-        wait_until(disconnected, timeout=10, lock=mininode_lock)
-
-    def wait_for_verack(self):
-        return all(node.wait_for_verack() for node in self.p2p_connections)
-
-    def wait_for_pings(self, counter):
-        def received_pongs():
-            return all(node.received_ping_response(counter) for node in self.p2p_connections)
-        wait_until(received_pongs, lock=mininode_lock)
-
-    # sync_blocks: Wait for all connections to request the blockhash given
-    # then send get_headers to find out the tip of each node, and synchronize
-    # the response by using a ping (and waiting for pong with same nonce).
-    def sync_blocks(self, blockhash, num_blocks):
-        def blocks_requested():
-            return all(
-                blockhash in node.block_request_map and node.block_request_map[blockhash]
-                for node in self.p2p_connections
-            )
-
-        # --> error if not requested
-        wait_until(blocks_requested, attempts=20*num_blocks, lock=mininode_lock)
-
-        # Send getheaders message
-        [ c.send_getheaders() for c in self.p2p_connections ]
-
-        # Send ping and wait for response -- synchronization hack
-        [ c.send_ping(self.ping_counter) for c in self.p2p_connections ]
-        self.wait_for_pings(self.ping_counter)
-        self.ping_counter += 1
-
-    # Analogous to sync_block (see above)
-    def sync_transaction(self, txhash, num_events):
-        # Wait for nodes to request transaction (50ms sleep * 20 tries * num_events)
-        def transaction_requested():
-            return all(
-                txhash in node.tx_request_map and node.tx_request_map[txhash]
-                for node in self.p2p_connections
-            )
-
-        # --> error if not requested
-        wait_until(transaction_requested, attempts=20*num_events, lock=mininode_lock)
-
-        # Get the mempool
-        [ c.send_mempool() for c in self.p2p_connections ]
-
-        # Send ping and wait for response -- synchronization hack
-        [ c.send_ping(self.ping_counter) for c in self.p2p_connections ]
-        self.wait_for_pings(self.ping_counter)
-        self.ping_counter += 1
-
-        # Sort inv responses from each node
-        with mininode_lock:
-            [ c.lastInv.sort() for c in self.p2p_connections ]
-
-    # Verify that the tip of each connection all agree with each other, and
-    # with the expected outcome (if given)
-    def check_results(self, blockhash, outcome):
-        with mininode_lock:
-            for c in self.p2p_connections:
-                if outcome is None:
-                    if c.bestblockhash != self.p2p_connections[0].bestblockhash:
-                        return False
-                elif isinstance(outcome, RejectResult): # Check that block was rejected w/ code
-                    if c.bestblockhash == blockhash:
-                        return False
-                    if blockhash not in c.block_reject_map:
-                        logger.error('Block not in reject map: %064x' % (blockhash))
-                        return False
-                    if not outcome.match(c.block_reject_map[blockhash]):
-                        logger.error('Block rejected with %s instead of expected %s: %064x' % (c.block_reject_map[blockhash], outcome, blockhash))
-                        return False
-                elif ((c.bestblockhash == blockhash) != outcome):
-                    return False
-            return True
-
-    # Either check that the mempools all agree with each other, or that
-    # txhash's presence in the mempool matches the outcome specified.
-    # This is somewhat of a strange comparison, in that we're either comparing
-    # a particular tx to an outcome, or the entire mempools altogether;
-    # perhaps it would be useful to add the ability to check explicitly that
-    # a particular tx's existence in the mempool is the same across all nodes.
-    def check_mempool(self, txhash, outcome):
-        with mininode_lock:
-            for c in self.p2p_connections:
-                if outcome is None:
-                    # Make sure the mempools agree with each other
-                    if c.lastInv != self.p2p_connections[0].lastInv:
-                        return False
-                elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code
-                    if txhash in c.lastInv:
-                        return False
-                    if txhash not in c.tx_reject_map:
-                        logger.error('Tx not in reject map: %064x' % (txhash))
-                        return False
-                    if not outcome.match(c.tx_reject_map[txhash]):
-                        logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.tx_reject_map[txhash], outcome, txhash))
-                        return False
-                elif ((txhash in c.lastInv) != outcome):
-                    return False
-            return True
-
-    def run(self):
-        # Wait until verack is received
-        self.wait_for_verack()
-
-        test_number = 0
-        tests = self.test_generator.get_tests()
-        for test_instance in tests:
-            test_number += 1
-            logger.info("Running test %d: %s line %s" % (test_number, tests.gi_code.co_filename, tests.gi_frame.f_lineno))
-            # We use these variables to keep track of the last block
-            # and last transaction in the tests, which are used
-            # if we're not syncing on every block or every tx.
-            [ block, block_outcome, tip ] = [ None, None, None ]
-            [ tx, tx_outcome ] = [ None, None ]
-            invqueue = []
-
-            for test_obj in test_instance.blocks_and_transactions:
-                b_or_t = test_obj[0]
-                outcome = test_obj[1]
-                # Determine if we're dealing with a block or tx
-                if isinstance(b_or_t, CBlock):  # Block test runner
-                    block = b_or_t
-                    block_outcome = outcome
-                    tip = block.sha256
-                    # each test_obj can have an optional third argument
-                    # to specify the tip we should compare with
-                    # (default is to use the block being tested)
-                    if len(test_obj) >= 3:
-                        tip = test_obj[2]
-
-                    # Add to shared block_store, set as current block
-                    # If there was an open getdata request for the block
-                    # previously, and we didn't have an entry in the
-                    # block_store, then immediately deliver, because the
-                    # node wouldn't send another getdata request while
-                    # the earlier one is outstanding.
-                    first_block_with_hash = True
-                    if self.block_store.get(block.sha256) is not None:
-                        first_block_with_hash = False
-                    with mininode_lock:
-                        self.block_store.add_block(block)
-                        for c in self.p2p_connections:
-                            if first_block_with_hash and block.sha256 in c.block_request_map and c.block_request_map[block.sha256] == True:
-                                # There was a previous request for this block hash
-                                # Most likely, we delivered a header for this block
-                                # but never had the block to respond to the getdata
-                                c.send_message(msg_block(block))
-                            else:
-                                c.block_request_map[block.sha256] = False
-                    # Either send inv's to each node and sync, or add
-                    # to invqueue for later inv'ing.
-                    if (test_instance.sync_every_block):
-                        # if we expect success, send inv and sync every block
-                        # if we expect failure, just push the block and see what happens.
-                        if outcome == True:
-                            [ c.send_inv(block) for c in self.p2p_connections ]
-                            self.sync_blocks(block.sha256, 1)
-                        else:
-                            [ c.send_message(msg_block(block)) for c in self.p2p_connections ]
-                            [ c.send_ping(self.ping_counter) for c in self.p2p_connections ]
-                            self.wait_for_pings(self.ping_counter)
-                            self.ping_counter += 1
-                        if (not self.check_results(tip, outcome)):
-                            raise AssertionError("Test failed at test %d" % test_number)
-                    else:
-                        invqueue.append(CInv(2, block.sha256))
-                elif isinstance(b_or_t, CBlockHeader):
-                    block_header = b_or_t
-                    self.block_store.add_header(block_header)
-                    [ c.send_header(block_header) for c in self.p2p_connections ]
-
-                else:  # Tx test runner
-                    assert(isinstance(b_or_t, CTransaction))
-                    tx = b_or_t
-                    tx_outcome = outcome
-                    # Add to shared tx store and clear map entry
-                    with mininode_lock:
-                        self.tx_store.add_transaction(tx)
-                        for c in self.p2p_connections:
-                            c.tx_request_map[tx.sha256] = False
-                    # Again, either inv to all nodes or save for later
-                    if (test_instance.sync_every_tx):
-                        [ c.send_inv(tx) for c in self.p2p_connections ]
-                        self.sync_transaction(tx.sha256, 1)
-                        if (not self.check_mempool(tx.sha256, outcome)):
-                            raise AssertionError("Test failed at test %d" % test_number)
-                    else:
-                        invqueue.append(CInv(1, tx.sha256))
-                # Ensure we're not overflowing the inv queue
-                if len(invqueue) == MAX_INV_SZ:
-                    [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ]
-                    invqueue = []
-
-            # Do final sync if we weren't syncing on every block or every tx.
-            if (not test_instance.sync_every_block and block is not None):
-                if len(invqueue) > 0:
-                    [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ]
-                    invqueue = []
-                self.sync_blocks(block.sha256, len(test_instance.blocks_and_transactions))
-                if (not self.check_results(tip, block_outcome)):
-                    raise AssertionError("Block test failed at test %d" % test_number)
-            if (not test_instance.sync_every_tx and tx is not None):
-                if len(invqueue) > 0:
-                    [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ]
-                    invqueue = []
-                self.sync_transaction(tx.sha256, len(test_instance.blocks_and_transactions))
-                if (not self.check_mempool(tx.sha256, tx_outcome)):
-                    raise AssertionError("Mempool test failed at test %d" % test_number)
-
-        [ c.disconnect_node() for c in self.p2p_connections ]
-        self.wait_for_disconnections()
-        self.block_store.close()
-        self.tx_store.close()
diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py
index d427f62856..543643f273 100755
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -432,35 +432,6 @@ class BitcoinTestFramework():
         for i in range(self.num_nodes):
             initialize_datadir(self.options.tmpdir, i)
 
-class ComparisonTestFramework(BitcoinTestFramework):
-    """Test framework for doing p2p comparison testing
-
-    Sets up some bitcoind binaries:
-    - 1 binary: test binary
-    - 2 binaries: 1 test binary, 1 ref binary
-    - n>2 binaries: 1 test binary, n-1 ref binaries"""
-
-    def set_test_params(self):
-        self.num_nodes = 2
-        self.setup_clean_chain = True
-
-    def add_options(self, parser):
-        parser.add_option("--testbinary", dest="testbinary",
-                          default=os.getenv("BITCOIND", "bitcoind"),
-                          help="bitcoind binary to test")
-        parser.add_option("--refbinary", dest="refbinary",
-                          default=os.getenv("BITCOIND", "bitcoind"),
-                          help="bitcoind binary to use for reference nodes (if any)")
-
-    def setup_network(self):
-        extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes
-        if hasattr(self, "extra_args"):
-            extra_args = self.extra_args
-        self.add_nodes(self.num_nodes, extra_args,
-                       binary=[self.options.testbinary] +
-                       [self.options.refbinary] * (self.num_nodes - 1))
-        self.start_nodes()
-
 class SkipTest(Exception):
     """This exception is raised to skip a test"""
     def __init__(self, message):