Merge bitcoin/bitcoin#19521: Coinstats Index

5f96d7d22d8e05876c6fc014e70488699950fe38 rpc: gettxoutsetinfo rejects hash_serialized_2 for specific height (Fabian Jahr)
23fe50436be641d7417152adc683192649ba206a test: Add test for coinstatsindex behavior in reorgs (Fabian Jahr)
90c966b0f3cfbd6bce5883f46d8527c6853a86a2 rpc: Allow gettxoutsetinfo and getblockstats for stale blocks (Fabian Jahr)
b9362392aef2689bc106c20925859ede555d082b index, rpc: Add use_index option for gettxoutsetinfo (Fabian Jahr)
bb7788b121a30489bc81a1f46dde6a9b19ae4ec1 test: Test coinstatsindex robustness across restarts (Fabian Jahr)
e0938c29099635150014ffc9bb0cafa8049ec55a test: Add tests for block_info in gettxoutsetinfo (Fabian Jahr)
2501576eccb08af80471c7b7b843b189ad6758c0 rpc, index: Add verbose amounts tracking to Coinstats index (Fabian Jahr)
655d929836a71af23d2035d2e2e99ad8b8c340c3 test: add coinstatsindex getindexinfo coverage, improve current tests (Jon Atack)
ca01bb8d689f93e1c7669b0ba7a4994c0206dabd rpc: Add Coinstats index to getindexinfo (Fabian Jahr)
57a026c30fef3138bb8db46e6865acb9dc2674f8 test: Add unit test for Coinstats index (Fabian Jahr)
6a4c0c09ab4d073a26c3c4a02783d5dcd88f6eef test: Add functional test for Coinstats index (Fabian Jahr)
3f166ecc125fce6ccd995687fa16572090a5d099 rpc: gettxoutsetinfo can be requested for specific blockheights (Fabian Jahr)
3c914d58ff323255b32e717d0ce28209ec0abdaa index: Coinstats index can be activated with command line flag (Fabian Jahr)
dd58a4de21469d6d848ae309edc47f558628221d index: Add Coinstats index (Fabian Jahr)
a8a46c4b3cfda4b95c92a36f8cebd3606377e57d refactor: Simplify ApplyStats and ApplyHash (Fabian Jahr)
9c8a265fd21a87228c18a1661df99fedc1866baf refactor: Pass hash_type to CoinsStats in stats object (Fabian Jahr)
2e2648a9021dfbb6e17dfa81472f057dacbc34e0 crypto: Make MuHash Remove method efficient (Fabian Jahr)

Pull request description:

  This is part of the coinstats index project tracked in #18000

  While the review of the new UTXO set hash algorithm (MuHash) takes longer recently #19328 was merged which added the possibility to run `gettxoutsetinfo` with a specific hash type. As the first type it added `hash_type=none` which skips the hashing of the UTXO set altogether. This alone did not make `gettxoutsetinfo` much faster but it allows the use of an index for the remaining coin statistics even before a new hashing algorithm has been added. Credit to Sjors for the idea to take this intermediate step.

  Features summary:
  - Users can start their node with the option `-coinstatsindex` which syncs the index in the background
  - After the index is synced the user can  use `gettxoutsetinfo` with `hash_type=none` or `hash_type=muhash` and will get the response instantly out of the index
  - The user can specify a height or block hash when calling `gettxoutsetinfo` to see coin statistics at a specific block height

ACKs for top commit:
  Sjors:
    re-tACK 5f96d7d22d8e05876c6fc014e70488699950fe38
  jonatack:
    Code review re-ACK 5f96d7d22d8e05876c6fc014e70488699950fe38 per `git range-diff 13d27b4 07201d3 5f96d7d`
  promag:
    Tested ACK 5f96d7d22d8e05876c6fc014e70488699950fe38. Light code review ACK 5f96d7d22d8e05876c6fc014e70488699950fe38.

Tree-SHA512: cbca78bee8e9605c19da4fbcd184625fb280200718396c694a56c7daab6f44ad23ca9fb5456d09f245d8b8d9659fdc2b3f3ce5e953c1c6cf4003dbc74c0463c2

5 files changed, 328 insertions, 16 deletions

diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py
new file mode 100755
index 0000000000..d3adde5cc5
--- /dev/null
+++ b/test/functional/feature_coinstatsindex.py
@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test coinstatsindex across nodes.
+
+Test that the values returned by gettxoutsetinfo are consistent
+between a node running the coinstatsindex and a node without
+the index.
+"""
+
+from decimal import Decimal
+
+from test_framework.blocktools import (
+    create_block,
+    create_coinbase,
+)
+from test_framework.messages import (
+    COIN,
+    COutPoint,
+    CTransaction,
+    CTxIn,
+    CTxOut,
+    ToHex,
+)
+from test_framework.script import (
+    CScript,
+    OP_FALSE,
+    OP_RETURN,
+)
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+    try_rpc,
+)
+
+class CoinStatsIndexTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.supports_cli = False
+        self.extra_args = [
+            [],
+            ["-coinstatsindex"]
+        ]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        self._test_coin_stats_index()
+        self._test_use_index_option()
+        self._test_reorg_index()
+        self._test_index_rejects_hash_serialized()
+
+    def block_sanity_check(self, block_info):
+        block_subsidy = 50
+        assert_equal(
+            block_info['prevout_spent'] + block_subsidy,
+            block_info['new_outputs_ex_coinbase'] + block_info['coinbase'] + block_info['unspendable']
+        )
+
+    def _test_coin_stats_index(self):
+        node = self.nodes[0]
+        index_node = self.nodes[1]
+        # Both none and muhash options allow the usage of the index
+        index_hash_options = ['none', 'muhash']
+
+        # Generate a normal transaction and mine it
+        node.generate(101)
+        address = self.nodes[0].get_deterministic_priv_key().address
+        node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True)
+        node.generate(1)
+
+        self.sync_blocks(timeout=120)
+
+        self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option")
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo))
+        res0 = node.gettxoutsetinfo('none')
+
+        # The fields 'disk_size' and 'transactions' do not exist on the index
+        del res0['disk_size'], res0['transactions']
+
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        for hash_option in index_hash_options:
+            res1 = index_node.gettxoutsetinfo(hash_option)
+            # The fields 'block_info' and 'total_unspendable_amount' only exist on the index
+            del res1['block_info'], res1['total_unspendable_amount']
+            res1.pop('muhash', None)
+
+            # Everything left should be the same
+            assert_equal(res1, res0)
+
+        self.log.info("Test that gettxoutsetinfo() can get fetch data on specific heights with index")
+
+        # Generate a new tip
+        node.generate(5)
+
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        for hash_option in index_hash_options:
+            # Fetch old stats by height
+            res2 = index_node.gettxoutsetinfo(hash_option, 102)
+            del res2['block_info'], res2['total_unspendable_amount']
+            res2.pop('muhash', None)
+            assert_equal(res0, res2)
+
+            # Fetch old stats by hash
+            res3 = index_node.gettxoutsetinfo(hash_option, res0['bestblock'])
+            del res3['block_info'], res3['total_unspendable_amount']
+            res3.pop('muhash', None)
+            assert_equal(res0, res3)
+
+            # It does not work without coinstatsindex
+            assert_raises_rpc_error(-8, "Querying specific block heights requires coinstatsindex", node.gettxoutsetinfo, hash_option, 102)
+
+        self.log.info("Test gettxoutsetinfo() with index and verbose flag")
+
+        for hash_option in index_hash_options:
+            # Genesis block is unspendable
+            res4 = index_node.gettxoutsetinfo(hash_option, 0)
+            assert_equal(res4['total_unspendable_amount'], 50)
+            assert_equal(res4['block_info'], {
+                'unspendable': 50,
+                'prevout_spent': 0,
+                'new_outputs_ex_coinbase': 0,
+                'coinbase': 0,
+                'unspendables': {
+                    'genesis_block': 50,
+                    'bip30': 0,
+                    'scripts': 0,
+                    'unclaimed_rewards': 0
+                }
+            })
+            self.block_sanity_check(res4['block_info'])
+
+            # Test an older block height that included a normal tx
+            res5 = index_node.gettxoutsetinfo(hash_option, 102)
+            assert_equal(res5['total_unspendable_amount'], 50)
+            assert_equal(res5['block_info'], {
+                'unspendable': 0,
+                'prevout_spent': 50,
+                'new_outputs_ex_coinbase': Decimal('49.99995560'),
+                'coinbase': Decimal('50.00004440'),
+                'unspendables': {
+                    'genesis_block': 0,
+                    'bip30': 0,
+                    'scripts': 0,
+                    'unclaimed_rewards': 0
+                }
+            })
+            self.block_sanity_check(res5['block_info'])
+
+        # Generate and send a normal tx with two outputs
+        tx1_inputs = []
+        tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42}
+        raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs)
+        funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1)
+        signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex'])
+        tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex'])
+
+        # Find the right position of the 21 BTC output
+        tx1_final = self.nodes[0].gettransaction(tx1_txid)
+        for output in tx1_final['details']:
+            if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive':
+                n = output['vout']
+
+        # Generate and send another tx with an OP_RETURN output (which is unspendable)
+        tx2 = CTransaction()
+        tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b''))
+        tx2.vout.append(CTxOut(int(20.99 * COIN), CScript([OP_RETURN] + [OP_FALSE]*30)))
+        tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx2))['hex']
+        self.nodes[0].sendrawtransaction(tx2_hex)
+
+        # Include both txs in a block
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        for hash_option in index_hash_options:
+            # Check all amounts were registered correctly
+            res6 = index_node.gettxoutsetinfo(hash_option, 108)
+            assert_equal(res6['total_unspendable_amount'], Decimal('70.98999999'))
+            assert_equal(res6['block_info'], {
+                'unspendable': Decimal('20.98999999'),
+                'prevout_spent': 111,
+                'new_outputs_ex_coinbase': Decimal('89.99993620'),
+                'coinbase': Decimal('50.01006381'),
+                'unspendables': {
+                    'genesis_block': 0,
+                    'bip30': 0,
+                    'scripts': Decimal('20.98999999'),
+                    'unclaimed_rewards': 0
+                }
+            })
+            self.block_sanity_check(res6['block_info'])
+
+        # Create a coinbase that does not claim full subsidy and also
+        # has two outputs
+        cb = create_coinbase(109, nValue=35)
+        cb.vout.append(CTxOut(5 * COIN, CScript([OP_FALSE])))
+        cb.rehash()
+
+        # Generate a block that includes previous coinbase
+        tip = self.nodes[0].getbestblockhash()
+        block_time = self.nodes[0].getblock(tip)['time'] + 1
+        block = create_block(int(tip, 16), cb, block_time)
+        block.solve()
+        self.nodes[0].submitblock(ToHex(block))
+        self.sync_all()
+
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        for hash_option in index_hash_options:
+            res7 = index_node.gettxoutsetinfo(hash_option, 109)
+            assert_equal(res7['total_unspendable_amount'], Decimal('80.98999999'))
+            assert_equal(res7['block_info'], {
+                'unspendable': 10,
+                'prevout_spent': 0,
+                'new_outputs_ex_coinbase': 0,
+                'coinbase': 40,
+                'unspendables': {
+                    'genesis_block': 0,
+                    'bip30': 0,
+                    'scripts': 0,
+                    'unclaimed_rewards': 10
+                }
+            })
+            self.block_sanity_check(res7['block_info'])
+
+        self.log.info("Test that the index is robust across restarts")
+
+        res8 = index_node.gettxoutsetinfo('muhash')
+        self.restart_node(1, extra_args=self.extra_args[1])
+        res9 = index_node.gettxoutsetinfo('muhash')
+        assert_equal(res8, res9)
+
+        index_node.generate(1)
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        res10 = index_node.gettxoutsetinfo('muhash')
+        assert(res8['txouts'] < res10['txouts'])
+
+    def _test_use_index_option(self):
+        self.log.info("Test use_index option for nodes running the index")
+
+        self.connect_nodes(0, 1)
+        self.nodes[0].waitforblockheight(110)
+        res = self.nodes[0].gettxoutsetinfo('muhash')
+        option_res = self.nodes[1].gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False)
+        del res['disk_size'], option_res['disk_size']
+        assert_equal(res, option_res)
+
+    def _test_reorg_index(self):
+        self.log.info("Test that index can handle reorgs")
+
+        # Generate two block, let the index catch up, then invalidate the blocks
+        index_node = self.nodes[1]
+        reorg_blocks = index_node.generatetoaddress(2, index_node.getnewaddress())
+        reorg_block = reorg_blocks[1]
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        res_invalid = index_node.gettxoutsetinfo('muhash')
+        index_node.invalidateblock(reorg_blocks[0])
+        assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110)
+
+        # Add two new blocks
+        block = index_node.generate(2)[1]
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False)
+
+        # Test that the result of the reorged block is not returned for its old block height
+        res2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112)
+        assert_equal(res["bestblock"], block)
+        assert_equal(res["muhash"], res2["muhash"])
+        assert(res["muhash"] != res_invalid["muhash"])
+
+        # Test that requesting reorged out block by hash is still returning correct results
+        res_invalid2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=reorg_block)
+        assert_equal(res_invalid2["muhash"], res_invalid["muhash"])
+        assert(res["muhash"] != res_invalid2["muhash"])
+
+        # Add another block, so we don't depend on reconsiderblock remembering which
+        # blocks were touched by invalidateblock
+        index_node.generate(1)
+
+        # Ensure that removing and re-adding blocks yields consistent results
+        block = index_node.getblockhash(99)
+        index_node.invalidateblock(block)
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        index_node.reconsiderblock(block)
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
+        res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112)
+        assert_equal(res2, res3)
+
+        self.log.info("Test that a node aware of stale blocks syncs them as well")
+        node = self.nodes[0]
+        # Ensure the node is aware of a stale block prior to restart
+        node.getblock(reorg_block)
+
+        self.restart_node(0, ["-coinstatsindex"])
+        self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash'))
+        assert_raises_rpc_error(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash', reorg_block)
+
+    def _test_index_rejects_hash_serialized(self):
+        self.log.info("Test that the rpc raises if the legacy hash is passed with the index")
+
+        msg = "hash_serialized_2 hash type cannot be queried for a specific block"
+        assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111)
+
+        for use_index in {True, False, None}:
+            assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111, use_index=use_index)
+
+
+if __name__ == '__main__':
+    CoinStatsIndexTest().main()
diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py
index a80fa596cd..52c8fa883d 100755
--- a/test/functional/rpc_misc.py
+++ b/test/functional/rpc_misc.py
@@ -69,25 +69,22 @@ class RpcMiscTest(BitcoinTestFramework):
         assert_equal(node.getindexinfo(), {})
 
         # Restart the node with indices and wait for them to sync
-        self.restart_node(0, ["-txindex", "-blockfilterindex"])
+        self.restart_node(0, ["-txindex", "-blockfilterindex", "-coinstatsindex"])
         self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values()))
 
         # Returns a list of all running indices by default
+        values = {"synced": True, "best_block_height": 200}
         assert_equal(
             node.getindexinfo(),
             {
-                "txindex": {"synced": True, "best_block_height": 200},
-                "basic block filter index": {"synced": True, "best_block_height": 200}
+                "txindex": values,
+                "basic block filter index": values,
+                "coinstatsindex": values,
             }
         )
-
         # Specifying an index by name returns only the status of that index
-        assert_equal(
-            node.getindexinfo("txindex"),
-            {
-                "txindex": {"synced": True, "best_block_height": 200},
-            }
-        )
+        for i in {"txindex", "basic block filter index", "coinstatsindex"}:
+            assert_equal(node.getindexinfo(i), {i: values})
 
         # Specifying an unknown index name returns an empty result
         assert_equal(node.getindexinfo("foo"), {})
diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py
index e691b63df6..d08e025178 100644
--- a/test/functional/test_framework/blocktools.py
+++ b/test/functional/test_framework/blocktools.py
@@ -115,7 +115,7 @@ def script_BIP34_coinbase_height(height):
     return CScript([CScriptNum(height)])
 
 
-def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0):
+def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50):
     """Create a coinbase transaction.
 
     If pubkey is passed in, the coinbase output will be a P2PK output;
@@ -126,10 +126,11 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0):
     coinbase = CTransaction()
     coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff))
     coinbaseoutput = CTxOut()
-    coinbaseoutput.nValue = 50 * COIN
-    halvings = int(height / 150)  # regtest
-    coinbaseoutput.nValue >>= halvings
-    coinbaseoutput.nValue += fees
+    coinbaseoutput.nValue = nValue * COIN
+    if nValue == 50:
+        halvings = int(height / 150)  # regtest
+        coinbaseoutput.nValue >>= halvings
+        coinbaseoutput.nValue += fees
     if pubkey is not None:
         coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG])
     else:
diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py
index 0ff4ee0a62..a89a26caea 100755
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -758,7 +758,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
 
             os.rmdir(cache_path('wallets'))  # Remove empty wallets dir
             for entry in os.listdir(cache_path()):
-                if entry not in ['chainstate', 'blocks']:  # Only keep chainstate and blocks folder
+                if entry not in ['chainstate', 'blocks', 'indexes']:  # Only indexes, chainstate and blocks folders
                     os.remove(cache_path(entry))
 
         for i in range(self.num_nodes):
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index bd58f2cd51..00527e78f1 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -281,6 +281,7 @@ BASE_SCRIPTS = [
     'rpc_scantxoutset.py',
     'feature_logging.py',
     'feature_anchors.py',
+    'feature_coinstatsindex.py',
     'p2p_node_network_limited.py',
     'p2p_permissions.py',
     'feature_blocksdir.py',