rpc, index: Add verbose amounts tracking to Coinstats index

6 files changed, 181 insertions, 10 deletions

diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp
index 7f59323b8c..c7c1f4b533 100644
--- a/src/index/coinstatsindex.cpp
+++ b/src/index/coinstatsindex.cpp
@@ -23,6 +23,15 @@ struct DBVal {
     uint64_t transaction_output_count;
     uint64_t bogo_size;
     CAmount total_amount;
+    CAmount total_subsidy;
+    CAmount block_unspendable_amount;
+    CAmount block_prevout_spent_amount;
+    CAmount block_new_outputs_ex_coinbase_amount;
+    CAmount block_coinbase_amount;
+    CAmount unspendables_genesis_block;
+    CAmount unspendables_bip30;
+    CAmount unspendables_scripts;
+    CAmount unspendables_unclaimed_rewards;
 
     SERIALIZE_METHODS(DBVal, obj)
     {
@@ -30,6 +39,15 @@ struct DBVal {
         READWRITE(obj.transaction_output_count);
         READWRITE(obj.bogo_size);
         READWRITE(obj.total_amount);
+        READWRITE(obj.total_subsidy);
+        READWRITE(obj.block_unspendable_amount);
+        READWRITE(obj.block_prevout_spent_amount);
+        READWRITE(obj.block_new_outputs_ex_coinbase_amount);
+        READWRITE(obj.block_coinbase_amount);
+        READWRITE(obj.unspendables_genesis_block);
+        READWRITE(obj.unspendables_bip30);
+        READWRITE(obj.unspendables_scripts);
+        READWRITE(obj.unspendables_unclaimed_rewards);
     }
 };
 
@@ -88,6 +106,8 @@ CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
 bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
 {
     CBlockUndo block_undo;
+    const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
+    m_total_subsidy += block_subsidy;
 
     // Ignore genesis block
     if (pindex->nHeight > 0) {
@@ -118,6 +138,8 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
 
             // Skip duplicate txid coinbase transactions (BIP30).
             if (is_bip30_block && tx->IsCoinBase()) {
+                m_block_unspendable_amount += block_subsidy;
+                m_unspendables_bip30 += block_subsidy;
                 continue;
             }
 
@@ -127,10 +149,20 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
                 COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)};
 
                 // Skip unspendable coins
-                if (coin.out.scriptPubKey.IsUnspendable()) continue;
+                if (coin.out.scriptPubKey.IsUnspendable()) {
+                    m_block_unspendable_amount += coin.out.nValue;
+                    m_unspendables_scripts += coin.out.nValue;
+                    continue;
+                }
 
                 m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin)));
 
+                if (tx->IsCoinBase()) {
+                    m_block_coinbase_amount += coin.out.nValue;
+                } else {
+                    m_block_new_outputs_ex_coinbase_amount += coin.out.nValue;
+                }
+
                 ++m_transaction_output_count;
                 m_total_amount += coin.out.nValue;
                 m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
@@ -146,19 +178,42 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
 
                     m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin)));
 
+                    m_block_prevout_spent_amount += coin.out.nValue;
+
                     --m_transaction_output_count;
                     m_total_amount -= coin.out.nValue;
                     m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
                 }
             }
         }
+    } else {
+        // genesis block
+        m_block_unspendable_amount += block_subsidy;
+        m_unspendables_genesis_block += block_subsidy;
     }
 
+    // If spent prevouts + block subsidy are still a higher amount than
+    // new outputs + coinbase + current unspendable amount this means
+    // the miner did not claim the full block reward. Unclaimed block
+    // rewards are also unspendable.
+    const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)};
+    m_block_unspendable_amount += unclaimed_rewards;
+    m_unspendables_unclaimed_rewards += unclaimed_rewards;
+
     std::pair<uint256, DBVal> value;
     value.first = pindex->GetBlockHash();
     value.second.transaction_output_count = m_transaction_output_count;
     value.second.bogo_size = m_bogo_size;
     value.second.total_amount = m_total_amount;
+    value.second.total_subsidy = m_total_subsidy;
+    value.second.block_unspendable_amount = m_block_unspendable_amount;
+    value.second.block_prevout_spent_amount = m_block_prevout_spent_amount;
+    value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount;
+    value.second.block_coinbase_amount = m_block_coinbase_amount;
+    value.second.unspendables_genesis_block = m_unspendables_genesis_block;
+    value.second.unspendables_bip30 = m_unspendables_bip30;
+    value.second.unspendables_scripts = m_unspendables_scripts;
+    value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards;
 
     uint256 out;
     m_muhash.Finalize(out);
@@ -261,6 +316,15 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co
     coins_stats.nTransactionOutputs = entry.transaction_output_count;
     coins_stats.nBogoSize = entry.bogo_size;
     coins_stats.nTotalAmount = entry.total_amount;
+    coins_stats.total_subsidy = entry.total_subsidy;
+    coins_stats.block_unspendable_amount = entry.block_unspendable_amount;
+    coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount;
+    coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
+    coins_stats.block_coinbase_amount = entry.block_coinbase_amount;
+    coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block;
+    coins_stats.unspendables_bip30 = entry.unspendables_bip30;
+    coins_stats.unspendables_scripts = entry.unspendables_scripts;
+    coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards;
 
     return true;
 }
@@ -289,6 +353,15 @@ bool CoinStatsIndex::Init()
             m_transaction_output_count = entry.transaction_output_count;
             m_bogo_size = entry.bogo_size;
             m_total_amount = entry.total_amount;
+            m_total_subsidy = entry.total_subsidy;
+            m_block_unspendable_amount = entry.block_unspendable_amount;
+            m_block_prevout_spent_amount = entry.block_prevout_spent_amount;
+            m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
+            m_block_coinbase_amount = entry.block_coinbase_amount;
+            m_unspendables_genesis_block = entry.unspendables_genesis_block;
+            m_unspendables_bip30 = entry.unspendables_bip30;
+            m_unspendables_scripts = entry.unspendables_scripts;
+            m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards;
         }
 
         return true;
@@ -303,6 +376,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
     CBlockUndo block_undo;
     std::pair<uint256, DBVal> read_out;
 
+    const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
+    m_total_subsidy -= block_subsidy;
+
     // Ignore genesis block
     if (pindex->nHeight > 0) {
         if (!UndoReadFromDisk(block_undo, pindex)) {
@@ -332,9 +408,23 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
             Coin coin{out, pindex->nHeight, tx->IsCoinBase()};
 
             // Skip unspendable coins
-            if (coin.out.scriptPubKey.IsUnspendable()) continue;
+            if (coin.out.scriptPubKey.IsUnspendable()) {
+                m_block_unspendable_amount -= coin.out.nValue;
+                m_unspendables_scripts -= coin.out.nValue;
+                continue;
+            }
 
             m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin)));
+
+            if (tx->IsCoinBase()) {
+                m_block_coinbase_amount -= coin.out.nValue;
+            } else {
+                m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue;
+            }
+
+            --m_transaction_output_count;
+            m_total_amount -= coin.out.nValue;
+            m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
         }
 
         // The coinbase tx has no undo data since no former output is spent
@@ -346,18 +436,37 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
                 COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
 
                 m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin)));
+
+                m_block_prevout_spent_amount -= coin.out.nValue;
+
+                m_transaction_output_count++;
+                m_total_amount += coin.out.nValue;
+                m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
             }
         }
     }
 
-    // Check that the rolled back internal value of muhash is consistent with the DB read out
+    const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)};
+    m_block_unspendable_amount -= unclaimed_rewards;
+    m_unspendables_unclaimed_rewards -= unclaimed_rewards;
+
+    // Check that the rolled back internal values are consistent with the DB read out
     uint256 out;
     m_muhash.Finalize(out);
     Assert(read_out.second.muhash == out);
 
-    m_transaction_output_count = read_out.second.transaction_output_count;
-    m_total_amount = read_out.second.total_amount;
-    m_bogo_size = read_out.second.bogo_size;
+    Assert(m_transaction_output_count == read_out.second.transaction_output_count);
+    Assert(m_total_amount == read_out.second.total_amount);
+    Assert(m_bogo_size == read_out.second.bogo_size);
+    Assert(m_total_subsidy == read_out.second.total_subsidy);
+    Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount);
+    Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount);
+    Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount);
+    Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount);
+    Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block);
+    Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30);
+    Assert(m_unspendables_scripts == read_out.second.unspendables_scripts);
+    Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards);
 
     return m_db->Write(DB_MUHASH, m_muhash);
 }
diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h
index 4d57e80770..6149f9b4b3 100644
--- a/src/index/coinstatsindex.h
+++ b/src/index/coinstatsindex.h
@@ -24,6 +24,15 @@ private:
     uint64_t m_transaction_output_count{0};
     uint64_t m_bogo_size{0};
     CAmount m_total_amount{0};
+    CAmount m_total_subsidy{0};
+    CAmount m_block_unspendable_amount{0};
+    CAmount m_block_prevout_spent_amount{0};
+    CAmount m_block_new_outputs_ex_coinbase_amount{0};
+    CAmount m_block_coinbase_amount{0};
+    CAmount m_unspendables_genesis_block{0};
+    CAmount m_unspendables_bip30{0};
+    CAmount m_unspendables_scripts{0};
+    CAmount m_unspendables_unclaimed_rewards{0};
 
     bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex);
 
diff --git a/src/node/coinstats.h b/src/node/coinstats.h
index e30b2778c5..ba565eba43 100644
--- a/src/node/coinstats.h
+++ b/src/node/coinstats.h
@@ -41,6 +41,17 @@ struct CCoinsStats
 
     bool from_index{false};
 
+    // Following values are only available from coinstats index
+    CAmount total_subsidy{0};
+    CAmount block_unspendable_amount{0};
+    CAmount block_prevout_spent_amount{0};
+    CAmount block_new_outputs_ex_coinbase_amount{0};
+    CAmount block_coinbase_amount{0};
+    CAmount unspendables_genesis_block{0};
+    CAmount unspendables_bip30{0};
+    CAmount unspendables_scripts{0};
+    CAmount unspendables_unclaimed_rewards{0};
+
     CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {}
 };
 
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index e6bf7f91b7..1067a7c4bb 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1113,8 +1113,23 @@ static RPCHelpMan gettxoutsetinfo()
                         {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"},
                         {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"},
                         {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"},
-                        {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"},
+                        {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"},
                         {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"},
+                        {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"},
+                        {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)",
+                        {
+                            {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""},
+                            {RPCResult::Type::STR_AMOUNT, "coinbase", ""},
+                            {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""},
+                            {RPCResult::Type::STR_AMOUNT, "unspendable", ""},
+                            {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories",
+                            {
+                                {RPCResult::Type::STR_AMOUNT, "genesis_block", ""},
+                                {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"},
+                                {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"},
+                                {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"},
+                            }}
+                        }},
                     }},
                 RPCExamples{
                     HelpExampleCli("gettxoutsetinfo", "") +
@@ -1130,9 +1145,7 @@ static RPCHelpMan gettxoutsetinfo()
 {
     UniValue ret(UniValue::VOBJ);
 
-    ::ChainstateActive().ForceFlushStateToDisk();
     CBlockIndex* pindex{nullptr};
-
     const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())};
     CCoinsStats stats{hash_type};
 
@@ -1147,6 +1160,7 @@ static RPCHelpMan gettxoutsetinfo()
         LOCK(::cs_main);
         coins_view = &active_chainstate.CoinsDB();
         blockman = &active_chainstate.m_blockman;
+        pindex = blockman->LookupBlockIndex(coins_view->GetBestBlock());
     }
 
     if (!request.params[1].isNull()) {
@@ -1168,11 +1182,34 @@ static RPCHelpMan gettxoutsetinfo()
         if (hash_type == CoinStatsHashType::MUHASH) {
               ret.pushKV("muhash", stats.hashSerialized.GetHex());
         }
+        ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount));
         if (!stats.from_index) {
             ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions));
             ret.pushKV("disk_size", stats.nDiskSize);
+        } else {
+            ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount));
+
+            CCoinsStats prev_stats{hash_type};
+
+            if (pindex->nHeight > 0) {
+                GetUTXOStats(coins_view, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), prev_stats, node.rpc_interruption_point, pindex->pprev);
+            }
+
+            UniValue block_info(UniValue::VOBJ);
+            block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount));
+            block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount));
+            block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount));
+            block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount));
+
+            UniValue unspendables(UniValue::VOBJ);
+            unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block));
+            unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30));
+            unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts));
+            unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards));
+            block_info.pushKV("unspendables", unspendables);
+
+            ret.pushKV("block_info", block_info);
         }
-        ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount));
     } else {
         if (g_coin_stats_index) {
             const IndexSummary summary{g_coin_stats_index->GetSummary()};
diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp
index 2b593cd10b..9ccfa36450 100644
--- a/src/rpc/client.cpp
+++ b/src/rpc/client.cpp
@@ -127,6 +127,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "gettxout", 1, "n" },
     { "gettxout", 2, "include_mempool" },
     { "gettxoutproof", 0, "txids" },
+    { "gettxoutsetinfo", 1, "hash_or_height" },
     { "lockunspent", 0, "unlock" },
     { "lockunspent", 1, "transactions" },
     { "send", 0, "outputs" },
diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py
index 9ff3a58510..ff4034176c 100755
--- a/test/functional/feature_coinstatsindex.py
+++ b/test/functional/feature_coinstatsindex.py
@@ -56,6 +56,8 @@ class CoinStatsIndexTest(BitcoinTestFramework):
         self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash'))
         for hash_option in index_hash_options:
             res1 = index_node.gettxoutsetinfo(hash_option)
+            # The fields 'block_info' and 'total_unspendable_amount' only exist on the index
+            del res1['block_info'], res1['total_unspendable_amount']
             res1.pop('muhash', None)
 
             # Everything left should be the same
@@ -70,11 +72,13 @@ class CoinStatsIndexTest(BitcoinTestFramework):
         for hash_option in index_hash_options:
             # Fetch old stats by height
             res2 = index_node.gettxoutsetinfo(hash_option, 102)
+            del res2['block_info'], res2['total_unspendable_amount']
             res2.pop('muhash', None)
             assert_equal(res0, res2)
 
             # Fetch old stats by hash
             res3 = index_node.gettxoutsetinfo(hash_option, res0['bestblock'])
+            del res3['block_info'], res3['total_unspendable_amount']
             res3.pop('muhash', None)
             assert_equal(res0, res3)