Merge bitcoin/bitcoin#21090: Default to NODE_WITNESS in nLocalServices

a806647d260132a00cd633160040625c7dd17803 [validation] Always include merkle root in coinbase commitment (Dhruv Mehta)
189128c220190a588500b8e74ee7ae47671b9558 [validation] Set witness script flag with p2sh for blocks (Dhruv Mehta)
ac82b99db77ec843af82dcdf040dfdbc98c8ff26 [p2p] remove redundant NODE_WITNESS checks (Dhruv Mehta)
6f8b198b8256a6703a6f5e592dfa77fa024a7035 [p2p] remove unused segwitheight=-1 option (Dhruv Mehta)
eba5b1cd6460c98e75d0422bd394e12af7f11e4c [test] remove or move tests using `-segwitheight=-1` (Dhruv Mehta)

Pull request description:

  Builds on #21009 and makes progress on remaining items in #17862

  Removing `RewindBlockIndex()` in #21009 allows the following:

  - removal of tests using `segwitheight=-1` in `p2p_segwit.py`.
  - move `test_upgrade_after_activation()` out of `p2p_segwit.py` reducing runtime
  - in turn, that allows us to drop support for `-segwitheight=-1`, which is only supported for that test.
  - that allows us to always set `NODE_WITNESS` in our local services. The only reason we don't do that is to support `-segwitheight=-1`.
  - that in turn allows us to drop all of the `GetLocalServices() & NODE_WITNESS` checks inside `net_processing.cpp`, since our local services would always include `NODE_WITNESS`

ACKs for top commit:
  mzumsande:
    Code-Review ACK a806647d260132a00cd633160040625c7dd17803
  laanwj:
    Code review ACK a806647d260132a00cd633160040625c7dd17803, nice cleanup
  jnewbery:
    utACK a806647d260132a00cd633160040625c7dd17803
  theStack:
    ACK a806647d260132a00cd633160040625c7dd17803

Tree-SHA512: 73e1a69d1d7eca1f5c38558ec6672decd0b60b16c2ef6134df6f6af71bb159e6eea160f9bb5ab0eb6723c6632d29509811e29469d0d87abbe9b69a2890fbc73e

4 files changed, 75 insertions, 79 deletions

diff --git a/test/functional/feature_presegwit_node_upgrade.py b/test/functional/feature_presegwit_node_upgrade.py
new file mode 100755
index 0000000000..0428588da3
--- /dev/null
+++ b/test/functional/feature_presegwit_node_upgrade.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test a pre-segwit node upgrading to segwit consensus"""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    softfork_active,
+)
+
+class SegwitUpgradeTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+        self.extra_args = [["-segwitheight=10"]]
+
+    def run_test(self):
+        """A pre-segwit node with insufficiently validated blocks needs to redownload blocks"""
+
+        self.log.info("Testing upgrade behaviour for pre-segwit node to segwit rules")
+        node = self.nodes[0]
+
+        # Node hasn't been used or connected yet
+        assert_equal(node.getblockcount(), 0)
+
+        assert not softfork_active(node, "segwit")
+
+        # Generate 8 blocks without witness data
+        node.generate(8)
+        assert_equal(node.getblockcount(), 8)
+
+        self.stop_node(0)
+        # Restarting the node (with segwit activation height set to 5) should result in a shutdown
+        # because the blockchain consists of 3 insufficiently validated blocks per segwit consensus rules.
+        node.assert_start_raises_init_error(
+                extra_args=["-segwitheight=5"],
+                expected_msg=": Witness data for blocks after height 5 requires validation. Please restart with -reindex..\nPlease restart with -reindex or -reindex-chainstate to recover.")
+
+        # As directed, the user restarts the node with -reindex
+        self.start_node(0, extra_args=["-reindex", "-segwitheight=5"])
+
+        # With the segwit consensus rules, the node is able to validate only up to block 4
+        assert_equal(node.getblockcount(), 4)
+
+        # The upgraded node should now have segwit activated
+        assert softfork_active(node, "segwit")
+
+
+if __name__ == '__main__':
+    SegwitUpgradeTest().main()
diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py
index ba467c1517..01fc02f27e 100755
--- a/test/functional/mining_basic.py
+++ b/test/functional/mining_basic.py
@@ -13,6 +13,7 @@ from decimal import Decimal
 
 from test_framework.blocktools import (
     create_coinbase,
+    get_witness_script,
     NORMAL_GBT_REQUEST_PARAMS,
     TIME_GENESIS_BLOCK,
 )
@@ -20,6 +21,7 @@ from test_framework.messages import (
     CBlock,
     CBlockHeader,
     BLOCK_HEADER_SIZE,
+    ser_uint256,
 )
 from test_framework.p2p import P2PDataStore
 from test_framework.test_framework import BitcoinTestFramework
@@ -49,6 +51,9 @@ class MiningTest(BitcoinTestFramework):
         self.setup_clean_chain = True
         self.supports_cli = False
 
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
     def mine_chain(self):
         self.log.info('Create some old blocks')
         for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600):
@@ -89,7 +94,21 @@ class MiningTest(BitcoinTestFramework):
         assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334'))
         assert_equal(mining_info['pooledtx'], 0)
 
-        # Mine a block to leave initial block download
+        self.log.info("getblocktemplate: Test default witness commitment")
+        txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16)
+        tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)
+
+        # Check that default_witness_commitment is present.
+        assert 'default_witness_commitment' in tmpl
+        witness_commitment = tmpl['default_witness_commitment']
+
+        # Check that default_witness_commitment is correct.
+        witness_root = CBlock.get_merkle_root([ser_uint256(0),
+                                               ser_uint256(txid)])
+        script = get_witness_script(witness_root, 0)
+        assert_equal(witness_commitment, script.hex())
+
+        # Mine a block to leave initial block download and clear the mempool
         node.generatetoaddress(1, node.get_deterministic_priv_key().address)
         tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)
         self.log.info("getblocktemplate: Test capability advertised")
diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py
index ead9d852fe..db96e6bdcf 100755
--- a/test/functional/p2p_segwit.py
+++ b/test/functional/p2p_segwit.py
@@ -9,11 +9,10 @@ import random
 import struct
 import time
 
-from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER
+from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER
 from test_framework.key import ECKey
 from test_framework.messages import (
     BIP125_SEQUENCE_NUMBER,
-    CBlock,
     CBlockHeader,
     CInv,
     COutPoint,
@@ -206,24 +205,17 @@ class TestP2PConn(P2PInterface):
 class SegWitTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
-        self.num_nodes = 3
+        self.num_nodes = 2
         # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation.
         self.extra_args = [
             ["-acceptnonstdtxn=1", "-segwitheight={}".format(SEGWIT_HEIGHT), "-whitelist=noban@127.0.0.1"],
             ["-acceptnonstdtxn=0", "-segwitheight={}".format(SEGWIT_HEIGHT)],
-            ["-acceptnonstdtxn=1", "-segwitheight=-1"],
         ]
         self.supports_cli = False
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
-    def setup_network(self):
-        self.setup_nodes()
-        self.connect_nodes(0, 1)
-        self.connect_nodes(0, 2)
-        self.sync_all()
-
     # Helper functions
 
     def build_next_block(self, version=4):
@@ -264,7 +256,6 @@ class SegWitTest(BitcoinTestFramework):
         self.test_non_witness_transaction()
         self.test_v0_outputs_arent_spendable()
         self.test_block_relay()
-        self.test_getblocktemplate_before_lockin()
         self.test_unnecessary_witness_before_segwit_activation()
         self.test_witness_tx_relay_before_segwit_activation()
         self.test_standardness_v0()
@@ -292,7 +283,6 @@ class SegWitTest(BitcoinTestFramework):
         self.test_signature_version_1()
         self.test_non_standard_witness_blinding()
         self.test_non_standard_witness()
-        self.test_upgrade_after_activation()
         self.test_witness_sigops()
         self.test_superfluous_witness()
         self.test_wtxid_relay()
@@ -482,11 +472,6 @@ class SegWitTest(BitcoinTestFramework):
         witness, and so can't be spent before segwit activation (the point at which
         blocks are permitted to contain witnesses)."""
 
-        # node2 doesn't need to be connected for this test.
-        # (If it's connected, node0 may propagate an invalid block to it over
-        # compact blocks and the nodes would have inconsistent tips.)
-        self.disconnect_nodes(0, 2)
-
         # Create two outputs, a p2wsh and p2sh-p2wsh
         witness_program = CScript([OP_TRUE])
         script_pubkey = script_to_p2wsh_script(witness_program)
@@ -544,38 +529,10 @@ class SegWitTest(BitcoinTestFramework):
             # TODO: support multiple acceptable reject reasons.
             test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False)
 
-        self.connect_nodes(0, 2)
-
         self.utxo.pop(0)
         self.utxo.append(UTXO(txid, 2, value))
 
     @subtest  # type: ignore
-    def test_getblocktemplate_before_lockin(self):
-        txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16)
-
-        for node in [self.nodes[0], self.nodes[2]]:
-            gbt_results = node.getblocktemplate({"rules": ["segwit"]})
-            if node == self.nodes[2]:
-                # If this is a non-segwit node, we should not get a witness
-                # commitment.
-                assert 'default_witness_commitment' not in gbt_results
-            else:
-                # For segwit-aware nodes, check the witness
-                # commitment is correct.
-                assert 'default_witness_commitment' in gbt_results
-                witness_commitment = gbt_results['default_witness_commitment']
-
-                # Check that default_witness_commitment is present.
-                witness_root = CBlock.get_merkle_root([ser_uint256(0),
-                                                       ser_uint256(txid)])
-                script = get_witness_script(witness_root, 0)
-                assert_equal(witness_commitment, script.hex())
-
-        # Clear out the mempool
-        self.nodes[0].generate(1)
-        self.sync_blocks()
-
-    @subtest  # type: ignore
     def test_witness_tx_relay_before_segwit_activation(self):
 
         # Generate a transaction that doesn't require a witness, but send it
@@ -1928,39 +1885,6 @@ class SegWitTest(BitcoinTestFramework):
         self.utxo.pop(0)
 
     @subtest  # type: ignore
-    def test_upgrade_after_activation(self):
-        """Test the behavior of starting up a segwit-aware node after the softfork has activated."""
-
-        # All nodes are caught up and node 2 is a pre-segwit node that will soon upgrade.
-        for n in range(2):
-            assert_equal(self.nodes[n].getblockcount(), self.nodes[2].getblockcount())
-            assert softfork_active(self.nodes[n], "segwit")
-        assert SEGWIT_HEIGHT < self.nodes[2].getblockcount()
-        assert 'segwit' not in self.nodes[2].getblockchaininfo()['softforks']
-
-        # Restarting node 2 should result in a shutdown because the blockchain consists of
-        # insufficiently validated blocks per segwit consensus rules.
-        self.stop_node(2)
-        self.nodes[2].assert_start_raises_init_error(
-            extra_args=[f"-segwitheight={SEGWIT_HEIGHT}"],
-            expected_msg=f": Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex..\nPlease restart with -reindex or -reindex-chainstate to recover.",
-        )
-
-        # As directed, the user restarts the node with -reindex
-        self.start_node(2, extra_args=["-reindex", f"-segwitheight={SEGWIT_HEIGHT}"])
-
-        # With the segwit consensus rules, the node is able to validate only up to SEGWIT_HEIGHT - 1
-        assert_equal(self.nodes[2].getblockcount(), SEGWIT_HEIGHT - 1)
-        self.connect_nodes(0, 2)
-
-        # We reconnect more than 100 blocks, give it plenty of time
-        # sync_blocks() also verifies the best block hash is the same for all nodes
-        self.sync_blocks(timeout=240)
-
-        # The upgraded node should now have segwit activated
-        assert softfork_active(self.nodes[2], "segwit")
-
-    @subtest  # type: ignore
     def test_witness_sigops(self):
         """Test sigop counting is correct inside witnesses."""
 
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 725706947f..32da2202db 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -297,6 +297,7 @@ BASE_SCRIPTS = [
     'wallet_startup.py',
     'p2p_i2p_ports.py',
     'feature_config_args.py',
+    'feature_presegwit_node_upgrade.py',
     'feature_settings.py',
     'rpc_getdescriptorinfo.py',
     'rpc_addresses_deprecation.py',