From 2e2648a9021dfbb6e17dfa81472f057dacbc34e0 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Thu, 25 Feb 2021 19:09:12 +0100 Subject: crypto: Make MuHash Remove method efficient Division of MuHash objects are very expensive and multiplication relatively cheap. The whole idea of introducing and tracking numerator and denominators seperately as a representation of the internal state was so that divisions would be rare. So using divison in the Remove method did not make any sense and was just a silly mistake which is corrected here. --- src/crypto/muhash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index e5a0d4cb9c..a2b769cd56 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -341,6 +341,6 @@ MuHash3072& MuHash3072::Insert(Span in) noexcept { } MuHash3072& MuHash3072::Remove(Span in) noexcept { - m_numerator.Divide(ToNum3072(in)); + m_denominator.Multiply(ToNum3072(in)); return *this; } -- cgit v1.2.3 From 9c8a265fd21a87228c18a1661df99fedc1866baf Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 21 Feb 2021 23:24:28 +0100 Subject: refactor: Pass hash_type to CoinsStats in stats object --- src/node/coinstats.cpp | 5 ++--- src/node/coinstats.h | 5 ++++- src/rpc/blockchain.cpp | 13 +++++++------ src/test/fuzz/coins_view.cpp | 4 ++-- src/validation.cpp | 4 ++-- 5 files changed, 17 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index f8f0fff43f..c52b718b22 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -86,7 +86,6 @@ static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, con template static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function& interruption_point) { - stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); assert(pcursor); @@ -129,9 +128,9 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& return true; } -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function& interruption_point) +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point) { - switch (hash_type) { + switch (stats.m_hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); return GetUTXOStats(view, blockman, stats, ss, interruption_point); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 975651dcc4..85896a2a1d 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -23,6 +23,7 @@ enum class CoinStatsHashType { struct CCoinsStats { + CoinStatsHashType m_hash_type; int nHeight{0}; uint256 hashBlock{}; uint64_t nTransactions{0}; @@ -34,9 +35,11 @@ struct CCoinsStats //! The number of coins contained. uint64_t coins_count{0}; + + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e7fd97ee1f..e06f4ec15b 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1093,14 +1093,14 @@ static RPCHelpMan gettxoutsetinfo() { UniValue ret(UniValue::VOBJ); - CCoinsStats stats; + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; + CCoinsStats stats{hash_type}; + NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); CChainState& active_chainstate = chainman.ActiveChainstate(); active_chainstate.ForceFlushStateToDisk(); - const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; - CCoinsView* coins_view; BlockManager* blockman; { @@ -1108,7 +1108,8 @@ static RPCHelpMan gettxoutsetinfo() coins_view = &active_chainstate.CoinsDB(); blockman = &active_chainstate.m_blockman; } - if (GetUTXOStats(coins_view, *blockman, stats, hash_type, node.rpc_interruption_point)) { + + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); @@ -2491,7 +2492,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) { std::unique_ptr pcursor; - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::NONE}; CBlockIndex* tip; { @@ -2511,7 +2512,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil chainstate.ForceFlushStateToDisk(); - if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b21d2eae79..21dc80cc8d 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -258,10 +258,10 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); }, [&] { - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED); + (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/validation.cpp b/src/validation.cpp index 332cb581b8..3ad53fbcb6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5285,14 +5285,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( return false; } - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; // As above, okay to immediately release cs_main here since no other context knows // about the snapshot_chainstate. CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); - if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) { + if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, breakpoint_fnc)) { LogPrintf("[snapshot] failed to generate coins stats\n"); return false; } -- cgit v1.2.3 From a8a46c4b3cfda4b95c92a36f8cebd3606377e57d Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 21 Feb 2021 23:09:50 +0100 Subject: refactor: Simplify ApplyStats and ApplyHash --- src/node/coinstats.cpp | 75 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 36 deletions(-) (limited to 'src') diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index c52b718b22..b17a475ca3 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -26,36 +26,6 @@ static uint64_t GetBogoSize(const CScript& scriptPubKey) scriptPubKey.size() /* scriptPubKey */; } -static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map& outputs, std::map::const_iterator it) -{ - if (it == outputs.begin()) { - ss << hash; - ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); - } - - ss << VARINT(it->first + 1); - ss << it->second.out.scriptPubKey; - ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - - if (it == std::prev(outputs.end())) { - ss << VARINT(0u); - } -} - -static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map& outputs, std::map::const_iterator it) {} - -static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map& outputs, std::map::const_iterator it) -{ - COutPoint outpoint = COutPoint(hash, it->first); - Coin coin = it->second; - - CDataStream ss(SER_DISK, PROTOCOL_VERSION); - ss << outpoint; - ss << static_cast(coin.nHeight * 2 + coin.fCoinBase); - ss << coin.out; - muhash.Insert(MakeUCharSpan(ss)); -} - //! Warning: be very careful when changing this! assumeutxo and UTXO snapshot //! validation commitments are reliant on the hash constructed by this //! function. @@ -68,14 +38,45 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has //! It is also possible, though very unlikely, that a change in this //! construction could cause a previously invalid (and potentially malicious) //! UTXO snapshot to be considered valid. -template -static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map& outputs) +static void ApplyHash(CHashWriter& ss, const uint256& hash, const std::map& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + if (it == outputs.begin()) { + ss << hash; + ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); + } + + ss << VARINT(it->first + 1); + ss << it->second.out.scriptPubKey; + ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + + if (it == std::prev(outputs.end())) { + ss << VARINT(0u); + } + } +} + +static void ApplyHash(std::nullptr_t, const uint256& hash, const std::map& outputs) {} + +static void ApplyHash(MuHash3072& muhash, const uint256& hash, const std::map& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + COutPoint outpoint = COutPoint(hash, it->first); + Coin coin = it->second; + + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + ss << outpoint; + ss << static_cast(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + muhash.Insert(MakeUCharSpan(ss)); + } +} + +static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map& outputs) { assert(!outputs.empty()); stats.nTransactions++; for (auto it = outputs.begin(); it != outputs.end(); ++it) { - ApplyHash(stats, hash_obj, hash, outputs, it); - stats.nTransactionOutputs++; stats.nTotalAmount += it->second.out.nValue; stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); @@ -107,7 +108,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -119,7 +121,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); } FinalizeHash(hash_obj, stats); -- cgit v1.2.3 From dd58a4de21469d6d848ae309edc47f558628221d Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Fri, 24 Jan 2020 18:56:47 +0100 Subject: index: Add Coinstats index The index holds the values previously calculated in coinstats.cpp for each block, representing the state of the UTXO set at each height. --- src/Makefile.am | 2 + src/index/base.h | 2 + src/index/coinstatsindex.cpp | 363 +++++++++++++++++++++++++++++++++++++++++++ src/index/coinstatsindex.h | 52 +++++++ src/node/coinstats.cpp | 19 ++- src/node/coinstats.h | 7 + 6 files changed, 437 insertions(+), 8 deletions(-) create mode 100644 src/index/coinstatsindex.cpp create mode 100644 src/index/coinstatsindex.h (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index ddeccd85ea..73d4edb77a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -152,6 +152,7 @@ BITCOIN_CORE_H = \ i2p.h \ index/base.h \ index/blockfilterindex.h \ + index/coinstatsindex.h \ index/disktxpos.h \ index/txindex.h \ indirectmap.h \ @@ -318,6 +319,7 @@ libbitcoin_server_a_SOURCES = \ i2p.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ + index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ mapport.cpp \ diff --git a/src/index/base.h b/src/index/base.h index ac3c429a5a..d887620524 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -81,6 +81,8 @@ protected: void ChainStateFlushed(const CBlockLocator& locator) override; + const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); }; + /// Initialize internal state from the database and block index. virtual bool Init(); diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp new file mode 100644 index 0000000000..7f59323b8c --- /dev/null +++ b/src/index/coinstatsindex.cpp @@ -0,0 +1,363 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr char DB_BLOCK_HASH = 's'; +static constexpr char DB_BLOCK_HEIGHT = 't'; +static constexpr char DB_MUHASH = 'M'; + +namespace { + +struct DBVal { + uint256 muhash; + uint64_t transaction_output_count; + uint64_t bogo_size; + CAmount total_amount; + + SERIALIZE_METHODS(DBVal, obj) + { + READWRITE(obj.muhash); + READWRITE(obj.transaction_output_count); + READWRITE(obj.bogo_size); + READWRITE(obj.total_amount); + } +}; + +struct DBHeightKey { + int height; + + explicit DBHeightKey(int height_in) : height(height_in) {} + + template + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template + void Unserialize(Stream& s) + { + char prefix{static_cast(ser_readdata8(s))}; + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 block_hash; + + explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} + + SERIALIZE_METHODS(DBHashKey, obj) + { + char prefix{DB_BLOCK_HASH}; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key"); + } + + READWRITE(obj.block_hash); + } +}; + +}; // namespace + +std::unique_ptr g_coin_stats_index; + +CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) +{ + fs::path path{GetDataDir() / "indexes" / "coinstats"}; + fs::create_directories(path); + + m_db = std::make_unique(path / "db", n_cache_size, f_memory, f_wipe); +} + +bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // TODO: Deduplicate BIP30 related code + bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + // Skip duplicate txid coinbase transactions (BIP30). + if (is_bip30_block && tx->IsCoinBase()) { + continue; + } + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + COutPoint outpoint{tx->GetHash(), static_cast(j)}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) continue; + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + ++m_transaction_output_count; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); + } + } + } + } + + std::pair value; + value.first = pindex->GetBlockHash(); + value.second.transaction_output_count = m_transaction_output_count; + value.second.bogo_size = m_bogo_size; + value.second.total_amount = m_total_amount; + + uint256 out; + m_muhash.Finalize(out); + value.second.muhash = out; + + return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash); +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key{start_height}; + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr db_it(m_db->NewIterator()); + + // During a reorg, we need to copy all hash digests for blocks that are + // getting disconnected from the height index to the hash index so we can + // still find them when the height index entries are overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + if (!m_db->WriteBatch(batch)) return false; + + { + LOCK(cs_main); + CBlockIndex* iter_tip{g_chainman.m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const auto& consensus_params{Params().GetConsensus()}; + + do { + CBlock block; + + if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + return error("%s: Failed to read block %s from disk", + __func__, iter_tip->GetBlockHash().ToString()); + } + + ReverseBlock(block, iter_tip); + + iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); + } while (new_tip != iter_tip); + } + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value + // there matches the block hash. This should be the case if the block is on + // the active chain. + std::pair read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the + // result will be stored in the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const +{ + DBVal entry; + if (!LookUpOne(*m_db, block_index, entry)) { + return false; + } + + coins_stats.hashSerialized = entry.muhash; + coins_stats.nTransactionOutputs = entry.transaction_output_count; + coins_stats.nBogoSize = entry.bogo_size; + coins_stats.nTotalAmount = entry.total_amount; + + return true; +} + +bool CoinStatsIndex::Init() +{ + if (!m_db->Read(DB_MUHASH, m_muhash)) { + // Check that the cause of the read failure is that the key does not + // exist. Any other errors indicate database corruption or a disk + // failure, and starting the index would cause further corruption. + if (m_db->Exists(DB_MUHASH)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + } + + if (BaseIndex::Init()) { + const CBlockIndex* pindex{CurrentIndex()}; + + if (pindex) { + DBVal entry; + if (!LookUpOne(*m_db, pindex, entry)) { + return false; + } + + m_transaction_output_count = entry.transaction_output_count; + m_bogo_size = entry.bogo_size; + m_total_amount = entry.total_amount; + } + + return true; + } + + return false; +} + +// Reverse a single block as part of a reorg +bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + std::pair read_out; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + } + + // Remove the new UTXOs that were created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + COutPoint outpoint{tx->GetHash(), static_cast(j)}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) continue; + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + } + } + } + + // Check that the rolled back internal value of muhash is consistent with the DB read out + uint256 out; + m_muhash.Finalize(out); + Assert(read_out.second.muhash == out); + + m_transaction_output_count = read_out.second.transaction_output_count; + m_total_amount = read_out.second.total_amount; + m_bogo_size = read_out.second.bogo_size; + + return m_db->Write(DB_MUHASH, m_muhash); +} diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h new file mode 100644 index 0000000000..4d57e80770 --- /dev/null +++ b/src/index/coinstatsindex.h @@ -0,0 +1,52 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_COINSTATSINDEX_H +#define BITCOIN_INDEX_COINSTATSINDEX_H + +#include +#include +#include +#include +#include + +/** + * CoinStatsIndex maintains statistics on the UTXO set. + */ +class CoinStatsIndex final : public BaseIndex +{ +private: + std::string m_name; + std::unique_ptr m_db; + + MuHash3072 m_muhash; + uint64_t m_transaction_output_count{0}; + uint64_t m_bogo_size{0}; + CAmount m_total_amount{0}; + + bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); + +protected: + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return "coinstatsindex"; } + +public: + // Constructs the index, which becomes available to be queried. + explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + // Look up stats for a specific block using CBlockIndex + bool LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const; +}; + +/// The global UTXO set hash object. +extern std::unique_ptr g_coin_stats_index; + +#endif // BITCOIN_INDEX_COINSTATSINDEX_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index b17a475ca3..a9af53ca80 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -16,14 +16,22 @@ #include // Database-independent metric indicating the UTXO set size -static uint64_t GetBogoSize(const CScript& scriptPubKey) +uint64_t GetBogoSize(const CScript& script_pub_key) { return 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + 2 /* scriptPubKey len */ + - scriptPubKey.size() /* scriptPubKey */; + script_pub_key.size() /* scriptPubKey */; +} + +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) { + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + ss << outpoint; + ss << static_cast(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + return ss; } //! Warning: be very careful when changing this! assumeutxo and UTXO snapshot @@ -63,12 +71,7 @@ static void ApplyHash(MuHash3072& muhash, const uint256& hash, const std::mapfirst); Coin coin = it->second; - - CDataStream ss(SER_DISK, PROTOCOL_VERSION); - ss << outpoint; - ss << static_cast(coin.nHeight * 2 + coin.fCoinBase); - ss << coin.out; - muhash.Insert(MakeUCharSpan(ss)); + muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); } } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 85896a2a1d..826df2fd73 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -7,6 +7,9 @@ #define BITCOIN_NODE_COINSTATS_H #include +#include +#include +#include #include #include @@ -42,4 +45,8 @@ struct CCoinsStats //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point = {}); +uint64_t GetBogoSize(const CScript& script_pub_key); + +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); + #endif // BITCOIN_NODE_COINSTATS_H -- cgit v1.2.3 From 3c914d58ff323255b32e717d0ce28209ec0abdaa Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Fri, 24 Jan 2020 18:58:47 +0100 Subject: index: Coinstats index can be activated with command line flag --- src/init.cpp | 20 ++++++++++++++++++-- src/node/coinstats.cpp | 13 ++++++++++--- src/rpc/blockchain.cpp | 8 ++++++++ src/validation.h | 1 + 4 files changed, 37 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/init.cpp b/src/init.cpp index 07e882c9df..f8265b5db2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -167,6 +168,9 @@ void Interrupt(NodeContext& node) g_txindex->Interrupt(); } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); + if (g_coin_stats_index) { + g_coin_stats_index->Interrupt(); + } } void Shutdown(NodeContext& node) @@ -252,6 +256,10 @@ void Shutdown(NodeContext& node) g_txindex->Stop(); g_txindex.reset(); } + if (g_coin_stats_index) { + g_coin_stats_index->Stop(); + g_coin_stats_index.reset(); + } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); DestroyAllBlockFilterIndexes(); @@ -390,6 +398,7 @@ void SetupServerArgs(NodeContext& node) #endif argsman.AddArg("-blockreconstructionextratxn=", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutset RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); @@ -406,7 +415,7 @@ void SetupServerArgs(NodeContext& node) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " + argsman.AddArg("-prune=", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -947,10 +956,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } - // if using block pruning, then disallow txindex + // if using block pruning, then disallow txindex and coinstatsindex if (args.GetArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) + return InitError(_("Prune mode is incompatible with -coinstatsindex.")); } // -bind and -whitebind can't be set when not listening @@ -1724,6 +1735,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) GetBlockFilterIndex(filter_type)->Start(); } + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { + g_coin_stats_index = std::make_unique(/* cache size */ 0, false, fReindex); + g_coin_stats_index->Start(); + } + // ********************************************************* Step 9: load wallet for (const auto& client : node.chain_clients) { if (!client->load()) { diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index a9af53ca80..dc9d32035d 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -92,13 +93,19 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& { std::unique_ptr pcursor(view->Cursor()); assert(pcursor); - stats.hashBlock = pcursor->GetBestBlock(); + + const CBlockIndex* pindex; { LOCK(cs_main); assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); - const CBlockIndex* block = blockman.LookupBlockIndex(stats.hashBlock); - stats.nHeight = Assert(block)->nHeight; + pindex = blockman.LookupBlockIndex(stats.hashBlock); + stats.nHeight = Assert(pindex)->nHeight; + } + + // Use CoinStatsIndex if it is available and a hash_type of Muhash or None was requested + if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index) { + return g_coin_stats_index->LookUpStats(pindex, stats); } PrepareHash(hash_obj, stats); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e06f4ec15b..dc6b7930b9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1124,6 +1125,13 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { + if (g_coin_stats_index) { + const IndexSummary summary{g_coin_stats_index->GetSummary()}; + + if (!summary.synced) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); + } + } throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } return ret; diff --git a/src/validation.h b/src/validation.h index de121ab46a..9a4b65ece6 100644 --- a/src/validation.h +++ b/src/validation.h @@ -79,6 +79,7 @@ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; -- cgit v1.2.3 From 3f166ecc125fce6ccd995687fa16572090a5d099 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Fri, 15 May 2020 16:14:07 +0200 Subject: rpc: gettxoutsetinfo can be requested for specific blockheights --- src/node/coinstats.cpp | 28 ++++++++-------- src/node/coinstats.h | 4 ++- src/rpc/blockchain.cpp | 91 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 76 insertions(+), 47 deletions(-) (limited to 'src') diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index dc9d32035d..eacd7e2ab5 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -89,22 +89,24 @@ static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map -static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function& interruption_point) +static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function& interruption_point, const CBlockIndex* pindex) { std::unique_ptr pcursor(view->Cursor()); assert(pcursor); - stats.hashBlock = pcursor->GetBestBlock(); - - const CBlockIndex* pindex; - { - LOCK(cs_main); - assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); - pindex = blockman.LookupBlockIndex(stats.hashBlock); - stats.nHeight = Assert(pindex)->nHeight; + + if (!pindex) { + { + LOCK(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + pindex = blockman.LookupBlockIndex(view->GetBestBlock()); + } } + stats.nHeight = Assert(pindex)->nHeight; + stats.hashBlock = pindex->GetBlockHash(); // Use CoinStatsIndex if it is available and a hash_type of Muhash or None was requested if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index) { + stats.from_index = true; return g_coin_stats_index->LookUpStats(pindex, stats); } @@ -141,19 +143,19 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& return true; } -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point) +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point, const CBlockIndex* pindex) { switch (stats.m_hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - return GetUTXOStats(view, blockman, stats, ss, interruption_point); + return GetUTXOStats(view, blockman, stats, ss, interruption_point, pindex); } case(CoinStatsHashType::MUHASH): { MuHash3072 muhash; - return GetUTXOStats(view, blockman, stats, muhash, interruption_point); + return GetUTXOStats(view, blockman, stats, muhash, interruption_point, pindex); } case(CoinStatsHashType::NONE): { - return GetUTXOStats(view, blockman, stats, nullptr, interruption_point); + return GetUTXOStats(view, blockman, stats, nullptr, interruption_point, pindex); } } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 826df2fd73..e30b2778c5 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -39,11 +39,13 @@ struct CCoinsStats //! The number of coins contained. uint64_t coins_count{0}; + bool from_index{false}; + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function& interruption_point = {}, const CBlockIndex* pindex = nullptr); uint64_t GetBogoSize(const CScript& script_pub_key); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index dc6b7930b9..e6bf7f91b7 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -140,6 +140,35 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } +CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { + LOCK(::cs_main); + CChain& active_chain = chainman.ActiveChain(); + + if (param.isNum()) { + const int height{param.get_int()}; + if (height < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); + } + const int current_tip{active_chain.Height()}; + if (height > current_tip) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); + } + + return active_chain[height]; + } else { + const uint256 hash{ParseHashV(param, "hash_or_height")}; + CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + + if (!pindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + if (!active_chain.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); + } + return pindex; + } +} + UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) { // Serialize passed information without accessing chain state of the active chain! @@ -1069,31 +1098,41 @@ static RPCHelpMan gettxoutsetinfo() { return RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" - "Note this call may take some time.\n", + "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The block hash or height of the target height (only available with coinstatsindex)", "", {"", "string or numeric"}}, }, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "height", "The block height (index) of the returned statistics"}, {RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at which these statistics are calculated"}, - {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, - {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, + {RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"}, {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, + {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, }}, RPCExamples{ - HelpExampleCli("gettxoutsetinfo", "") - + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleCli("gettxoutsetinfo", "") + + HelpExampleCli("gettxoutsetinfo", R"("none")") + + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleRpc("gettxoutsetinfo", "") + + HelpExampleRpc("gettxoutsetinfo", R"("none")") + + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + + HelpExampleRpc("gettxoutsetinfo", R"("none", "00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09")") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { UniValue ret(UniValue::VOBJ); + ::ChainstateActive().ForceFlushStateToDisk(); + CBlockIndex* pindex{nullptr}; + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; @@ -1110,10 +1149,17 @@ static RPCHelpMan gettxoutsetinfo() blockman = &active_chainstate.m_blockman; } - if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point)) { + if (!request.params[1].isNull()) { + if (!g_coin_stats_index) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Querying specific block heights requires coinstatsindex"); + } + + pindex = ParseHashOrHeight(request.params[1], chainman); + } + + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); - ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { @@ -1122,7 +1168,10 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::MUHASH) { ret.pushKV("muhash", stats.hashSerialized.GetHex()); } - ret.pushKV("disk_size", stats.nDiskSize); + if (!stats.from_index) { + ret.pushKV("transactions", static_cast(stats.nTransactions)); + ret.pushKV("disk_size", stats.nDiskSize); + } ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { if (g_coin_stats_index) { @@ -1918,31 +1967,7 @@ static RPCHelpMan getblockstats() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CChain& active_chain = chainman.ActiveChain(); - - CBlockIndex* pindex; - if (request.params[0].isNum()) { - const int height = request.params[0].get_int(); - const int current_tip = active_chain.Height(); - if (height < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); - } - if (height > current_tip) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); - } - - pindex = active_chain[height]; - } else { - const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = chainman.m_blockman.LookupBlockIndex(hash); - if (!pindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - if (!active_chain.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); - } - } - + CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; CHECK_NONFATAL(pindex != nullptr); std::set stats; -- cgit v1.2.3 From 57a026c30fef3138bb8db46e6865acb9dc2674f8 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Wed, 28 Aug 2019 15:58:53 -0400 Subject: test: Add unit test for Coinstats index --- src/Makefile.test.include | 1 + src/test/coinstatsindex_tests.cpp | 79 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/test/coinstatsindex_tests.cpp (limited to 'src') diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 570f011f7a..b546f1f55f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -80,6 +80,7 @@ BITCOIN_TESTS =\ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ test/coins_tests.cpp \ + test/coinstatsindex_tests.cpp \ test/compilerbug_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp new file mode 100644 index 0000000000..3fc7b72077 --- /dev/null +++ b/src/test/coinstatsindex_tests.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include + +#include + + +BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) + +BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) +{ + CoinStatsIndex coin_stats_index{1 << 20, true}; + + CCoinsStats coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = ChainActive().Tip(); + } + + // CoinStatsIndex should not be found before it is started. + BOOST_CHECK(!coin_stats_index.LookUpStats(block_index, coin_stats)); + + // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex + // is started. + BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); + + coin_stats_index.Start(); + + // Allow the CoinStatsIndex to catch up with the block index that is syncing + // in a background thread. + const auto timeout = GetTime() + 120s; + while (!coin_stats_index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(timeout > GetTime()); + UninterruptibleSleep(100ms); + } + + // Check that CoinStatsIndex works for genesis block. + const CBlockIndex* genesis_block_index; + { + LOCK(cs_main); + genesis_block_index = ChainActive().Genesis(); + } + BOOST_CHECK(coin_stats_index.LookUpStats(genesis_block_index, coin_stats)); + + // Check that CoinStatsIndex updates with new blocks. + coin_stats_index.LookUpStats(block_index, coin_stats); + + const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG}; + std::vector noTxns; + CreateAndProcessBlock(noTxns, script_pub_key); + + // Let the CoinStatsIndex to catch up again. + BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain()); + + CCoinsStats new_coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* new_block_index; + { + LOCK(cs_main); + new_block_index = ChainActive().Tip(); + } + coin_stats_index.LookUpStats(new_block_index, new_coin_stats); + + BOOST_CHECK(block_index != new_block_index); + + // Shutdown sequence (c.f. Shutdown() in init.cpp) + coin_stats_index.Stop(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() +} + +BOOST_AUTO_TEST_SUITE_END() -- cgit v1.2.3 From ca01bb8d689f93e1c7669b0ba7a4994c0206dabd Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sat, 22 Aug 2020 18:31:34 +0200 Subject: rpc: Add Coinstats index to getindexinfo --- src/rpc/misc.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 00a06260ea..dd495850d3 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -689,6 +690,10 @@ static RPCHelpMan getindexinfo() result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); } + if (g_coin_stats_index) { + result.pushKVs(SummaryToJSON(g_coin_stats_index->GetSummary(), index_name)); + } + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); }); -- cgit v1.2.3 From 2501576eccb08af80471c7b7b843b189ad6758c0 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sat, 22 Aug 2020 20:21:20 +0200 Subject: rpc, index: Add verbose amounts tracking to Coinstats index --- src/index/coinstatsindex.cpp | 121 ++++++++++++++++++++++++++++++++++++++++--- src/index/coinstatsindex.h | 9 ++++ src/node/coinstats.h | 11 ++++ src/rpc/blockchain.cpp | 45 ++++++++++++++-- src/rpc/client.cpp | 1 + 5 files changed, 177 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 7f59323b8c..c7c1f4b533 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -23,6 +23,15 @@ struct DBVal { uint64_t transaction_output_count; uint64_t bogo_size; CAmount total_amount; + CAmount total_subsidy; + CAmount block_unspendable_amount; + CAmount block_prevout_spent_amount; + CAmount block_new_outputs_ex_coinbase_amount; + CAmount block_coinbase_amount; + CAmount unspendables_genesis_block; + CAmount unspendables_bip30; + CAmount unspendables_scripts; + CAmount unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { @@ -30,6 +39,15 @@ struct DBVal { READWRITE(obj.transaction_output_count); READWRITE(obj.bogo_size); READWRITE(obj.total_amount); + READWRITE(obj.total_subsidy); + READWRITE(obj.block_unspendable_amount); + READWRITE(obj.block_prevout_spent_amount); + READWRITE(obj.block_new_outputs_ex_coinbase_amount); + READWRITE(obj.block_coinbase_amount); + READWRITE(obj.unspendables_genesis_block); + READWRITE(obj.unspendables_bip30); + READWRITE(obj.unspendables_scripts); + READWRITE(obj.unspendables_unclaimed_rewards); } }; @@ -88,6 +106,8 @@ CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) { CBlockUndo block_undo; + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy += block_subsidy; // Ignore genesis block if (pindex->nHeight > 0) { @@ -118,6 +138,8 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { + m_block_unspendable_amount += block_subsidy; + m_unspendables_bip30 += block_subsidy; continue; } @@ -127,10 +149,20 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) COutPoint outpoint{tx->GetHash(), static_cast(j)}; // Skip unspendable coins - if (coin.out.scriptPubKey.IsUnspendable()) continue; + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount += coin.out.nValue; + m_unspendables_scripts += coin.out.nValue; + continue; + } m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + if (tx->IsCoinBase()) { + m_block_coinbase_amount += coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; + } + ++m_transaction_output_count; m_total_amount += coin.out.nValue; m_bogo_size += GetBogoSize(coin.out.scriptPubKey); @@ -146,19 +178,42 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + m_block_prevout_spent_amount += coin.out.nValue; + --m_transaction_output_count; m_total_amount -= coin.out.nValue; m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); } } } + } else { + // genesis block + m_block_unspendable_amount += block_subsidy; + m_unspendables_genesis_block += block_subsidy; } + // If spent prevouts + block subsidy are still a higher amount than + // new outputs + coinbase + current unspendable amount this means + // the miner did not claim the full block reward. Unclaimed block + // rewards are also unspendable. + const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)}; + m_block_unspendable_amount += unclaimed_rewards; + m_unspendables_unclaimed_rewards += unclaimed_rewards; + std::pair value; value.first = pindex->GetBlockHash(); value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; + value.second.total_subsidy = m_total_subsidy; + value.second.block_unspendable_amount = m_block_unspendable_amount; + value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; + value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; + value.second.block_coinbase_amount = m_block_coinbase_amount; + value.second.unspendables_genesis_block = m_unspendables_genesis_block; + value.second.unspendables_bip30 = m_unspendables_bip30; + value.second.unspendables_scripts = m_unspendables_scripts; + value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); @@ -261,6 +316,15 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co coins_stats.nTransactionOutputs = entry.transaction_output_count; coins_stats.nBogoSize = entry.bogo_size; coins_stats.nTotalAmount = entry.total_amount; + coins_stats.total_subsidy = entry.total_subsidy; + coins_stats.block_unspendable_amount = entry.block_unspendable_amount; + coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; + coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + coins_stats.block_coinbase_amount = entry.block_coinbase_amount; + coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block; + coins_stats.unspendables_bip30 = entry.unspendables_bip30; + coins_stats.unspendables_scripts = entry.unspendables_scripts; + coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; return true; } @@ -289,6 +353,15 @@ bool CoinStatsIndex::Init() m_transaction_output_count = entry.transaction_output_count; m_bogo_size = entry.bogo_size; m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_block_unspendable_amount = entry.block_unspendable_amount; + m_block_prevout_spent_amount = entry.block_prevout_spent_amount; + m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + m_block_coinbase_amount = entry.block_coinbase_amount; + m_unspendables_genesis_block = entry.unspendables_genesis_block; + m_unspendables_bip30 = entry.unspendables_bip30; + m_unspendables_scripts = entry.unspendables_scripts; + m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; } return true; @@ -303,6 +376,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex CBlockUndo block_undo; std::pair read_out; + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy -= block_subsidy; + // Ignore genesis block if (pindex->nHeight > 0) { if (!UndoReadFromDisk(block_undo, pindex)) { @@ -332,9 +408,23 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; // Skip unspendable coins - if (coin.out.scriptPubKey.IsUnspendable()) continue; + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount -= coin.out.nValue; + m_unspendables_scripts -= coin.out.nValue; + continue; + } m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_block_coinbase_amount -= coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue; + } + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); } // The coinbase tx has no undo data since no former output is spent @@ -346,18 +436,37 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_block_prevout_spent_amount -= coin.out.nValue; + + m_transaction_output_count++; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); } } } - // Check that the rolled back internal value of muhash is consistent with the DB read out + const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)}; + m_block_unspendable_amount -= unclaimed_rewards; + m_unspendables_unclaimed_rewards -= unclaimed_rewards; + + // Check that the rolled back internal values are consistent with the DB read out uint256 out; m_muhash.Finalize(out); Assert(read_out.second.muhash == out); - m_transaction_output_count = read_out.second.transaction_output_count; - m_total_amount = read_out.second.total_amount; - m_bogo_size = read_out.second.bogo_size; + Assert(m_transaction_output_count == read_out.second.transaction_output_count); + Assert(m_total_amount == read_out.second.total_amount); + Assert(m_bogo_size == read_out.second.bogo_size); + Assert(m_total_subsidy == read_out.second.total_subsidy); + Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount); + Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount); + Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount); + Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount); + Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block); + Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30); + Assert(m_unspendables_scripts == read_out.second.unspendables_scripts); + Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards); return m_db->Write(DB_MUHASH, m_muhash); } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 4d57e80770..6149f9b4b3 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -24,6 +24,15 @@ private: uint64_t m_transaction_output_count{0}; uint64_t m_bogo_size{0}; CAmount m_total_amount{0}; + CAmount m_total_subsidy{0}; + CAmount m_block_unspendable_amount{0}; + CAmount m_block_prevout_spent_amount{0}; + CAmount m_block_new_outputs_ex_coinbase_amount{0}; + CAmount m_block_coinbase_amount{0}; + CAmount m_unspendables_genesis_block{0}; + CAmount m_unspendables_bip30{0}; + CAmount m_unspendables_scripts{0}; + CAmount m_unspendables_unclaimed_rewards{0}; bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index e30b2778c5..ba565eba43 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -41,6 +41,17 @@ struct CCoinsStats bool from_index{false}; + // Following values are only available from coinstats index + CAmount total_subsidy{0}; + CAmount block_unspendable_amount{0}; + CAmount block_prevout_spent_amount{0}; + CAmount block_new_outputs_ex_coinbase_amount{0}; + CAmount block_coinbase_amount{0}; + CAmount unspendables_genesis_block{0}; + CAmount unspendables_bip30{0}; + CAmount unspendables_scripts{0}; + CAmount unspendables_unclaimed_rewards{0}; + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e6bf7f91b7..1067a7c4bb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1113,8 +1113,23 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, - {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, + {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, + {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, + {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", + { + {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""}, + {RPCResult::Type::STR_AMOUNT, "coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "unspendable", ""}, + {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories", + { + {RPCResult::Type::STR_AMOUNT, "genesis_block", ""}, + {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"}, + {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"}, + {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"}, + }} + }}, }}, RPCExamples{ HelpExampleCli("gettxoutsetinfo", "") + @@ -1130,9 +1145,7 @@ static RPCHelpMan gettxoutsetinfo() { UniValue ret(UniValue::VOBJ); - ::ChainstateActive().ForceFlushStateToDisk(); CBlockIndex* pindex{nullptr}; - const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; @@ -1147,6 +1160,7 @@ static RPCHelpMan gettxoutsetinfo() LOCK(::cs_main); coins_view = &active_chainstate.CoinsDB(); blockman = &active_chainstate.m_blockman; + pindex = blockman->LookupBlockIndex(coins_view->GetBestBlock()); } if (!request.params[1].isNull()) { @@ -1168,11 +1182,34 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::MUHASH) { ret.pushKV("muhash", stats.hashSerialized.GetHex()); } + ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); if (!stats.from_index) { ret.pushKV("transactions", static_cast(stats.nTransactions)); ret.pushKV("disk_size", stats.nDiskSize); + } else { + ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount)); + + CCoinsStats prev_stats{hash_type}; + + if (pindex->nHeight > 0) { + GetUTXOStats(coins_view, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), prev_stats, node.rpc_interruption_point, pindex->pprev); + } + + UniValue block_info(UniValue::VOBJ); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount)); + + UniValue unspendables(UniValue::VOBJ); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards)); + block_info.pushKV("unspendables", unspendables); + + ret.pushKV("block_info", block_info); } - ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { if (g_coin_stats_index) { const IndexSummary summary{g_coin_stats_index->GetSummary()}; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 2b593cd10b..9ccfa36450 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -127,6 +127,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, + { "gettxoutsetinfo", 1, "hash_or_height" }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "send", 0, "outputs" }, -- cgit v1.2.3 From b9362392aef2689bc106c20925859ede555d082b Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 28 Feb 2021 19:27:00 +0100 Subject: index, rpc: Add use_index option for gettxoutsetinfo --- src/node/coinstats.cpp | 6 +++--- src/node/coinstats.h | 5 ++++- src/rpc/blockchain.cpp | 4 +++- src/rpc/client.cpp | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index eacd7e2ab5..38c1d29250 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -104,9 +104,9 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats.nHeight = Assert(pindex)->nHeight; stats.hashBlock = pindex->GetBlockHash(); - // Use CoinStatsIndex if it is available and a hash_type of Muhash or None was requested - if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index) { - stats.from_index = true; + // Use CoinStatsIndex if it is requested and available and a hash_type of Muhash or None was requested + if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index && stats.index_requested) { + stats.index_used = true; return g_coin_stats_index->LookUpStats(pindex, stats); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index ba565eba43..8be256edc9 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -39,7 +39,10 @@ struct CCoinsStats //! The number of coins contained. uint64_t coins_count{0}; - bool from_index{false}; + //! Signals if the coinstatsindex should be used (when available). + bool index_requested{true}; + //! Signals if the coinstatsindex was used to retrieve the statistics. + bool index_used{false}; // Following values are only available from coinstats index CAmount total_subsidy{0}; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1067a7c4bb..4a4c432a01 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1102,6 +1102,7 @@ static RPCHelpMan gettxoutsetinfo() { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The block hash or height of the target height (only available with coinstatsindex)", "", {"", "string or numeric"}}, + {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -1148,6 +1149,7 @@ static RPCHelpMan gettxoutsetinfo() CBlockIndex* pindex{nullptr}; const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsStats stats{hash_type}; + stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); @@ -1183,7 +1185,7 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("muhash", stats.hashSerialized.GetHex()); } ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); - if (!stats.from_index) { + if (!stats.index_used) { ret.pushKV("transactions", static_cast(stats.nTransactions)); ret.pushKV("disk_size", stats.nDiskSize); } else { diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 9ccfa36450..9c8582c7a3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -128,6 +128,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, { "gettxoutsetinfo", 1, "hash_or_height" }, + { "gettxoutsetinfo", 2, "use_index"}, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "send", 0, "outputs" }, -- cgit v1.2.3 From 90c966b0f3cfbd6bce5883f46d8527c6853a86a2 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Thu, 11 Mar 2021 00:52:46 +0100 Subject: rpc: Allow gettxoutsetinfo and getblockstats for stale blocks --- src/rpc/blockchain.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4a4c432a01..f8d7d94abf 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -162,9 +162,7 @@ CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainma if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - if (!active_chain.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); - } + return pindex; } } -- cgit v1.2.3 From 5f96d7d22d8e05876c6fc014e70488699950fe38 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Thu, 4 Mar 2021 01:37:50 +0100 Subject: rpc: gettxoutsetinfo rejects hash_serialized_2 for specific height --- src/rpc/blockchain.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f8d7d94abf..438b6ce6e6 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1168,6 +1168,10 @@ static RPCHelpMan gettxoutsetinfo() throw JSONRPCError(RPC_INVALID_PARAMETER, "Querying specific block heights requires coinstatsindex"); } + if (stats.m_hash_type == CoinStatsHashType::HASH_SERIALIZED) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); + } + pindex = ParseHashOrHeight(request.params[1], chainman); } -- cgit v1.2.3