diff options
Diffstat (limited to 'src/rpc/blockchain.cpp')
| -rw-r--r-- | src/rpc/blockchain.cpp | 548 |
1 files changed, 378 insertions, 170 deletions
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e785678614..360f24ec55 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -8,6 +8,7 @@ #include <blockfilter.h> #include <chain.h> #include <chainparams.h> +#include <chainparamsbase.h> #include <clientversion.h> #include <coins.h> #include <common/args.h> @@ -21,6 +22,7 @@ #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> +#include <interfaces/mining.h> #include <kernel/coinstats.h> #include <logging/timer.h> #include <net.h> @@ -55,26 +57,32 @@ #include <condition_variable> #include <memory> #include <mutex> +#include <optional> using kernel::CCoinsStats; using kernel::CoinStatsHashType; +using interfaces::Mining; using node::BlockManager; using node::NodeContext; using node::SnapshotMetadata; -using util::Join; using util::MakeUnorderedList; -using util::ToString; -struct CUpdatedBlock -{ - uint256 hash; - int height; -}; +std::tuple<std::unique_ptr<CCoinsViewCursor>, CCoinsStats, const CBlockIndex*> +PrepareUTXOSnapshot( + Chainstate& chainstate, + const std::function<void()>& interruption_point = {}) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -static GlobalMutex cs_blockchange; -static std::condition_variable cond_blockchange; -static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); +UniValue WriteUTXOSnapshot( + Chainstate& chainstate, + CCoinsViewCursor* pcursor, + CCoinsStats* maybe_stats, + const CBlockIndex* tip, + AutoFile& afile, + const fs::path& path, + const fs::path& temppath, + const std::function<void()>& interruption_point = {}); /* Calculate the difficulty for a given block index. */ @@ -185,8 +193,10 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, blockindex)}; - + bool have_undo{is_not_pruned && WITH_LOCK(::cs_main, return blockindex.nStatus & BLOCK_HAVE_UNDO)}; + if (have_undo && !blockman.UndoReadFromDisk(blockUndo, blockindex)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event."); + } for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i.e. i == 0) doesn't have undo data @@ -244,21 +254,12 @@ static RPCHelpMan getbestblockhash() }; } -void RPCNotifyBlockChange(const CBlockIndex* pindex) -{ - if(pindex) { - LOCK(cs_blockchange); - latestblock.hash = pindex->GetBlockHash(); - latestblock.height = pindex->nHeight; - } - cond_blockchange.notify_all(); -} - static RPCHelpMan waitfornewblock() { return RPCHelpMan{"waitfornewblock", - "\nWaits for a specific new block and returns useful info about it.\n" - "\nReturns the current block on timeout or exit.\n", + "\nWaits for any new block and returns useful info about it.\n" + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, }, @@ -277,17 +278,16 @@ static RPCHelpMan waitfornewblock() int timeout = 0; if (!request.params[0].isNull()) timeout = request.params[0].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - block = latestblock; - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); - else - cond_blockchange.wait(lock, [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + if (IsRPCRunning()) { + block = timeout ? miner.waitTipChanged(block.hash, std::chrono::milliseconds(timeout)) : miner.waitTipChanged(block.hash); } + UniValue ret(UniValue::VOBJ); ret.pushKV("hash", block.hash.GetHex()); ret.pushKV("height", block.height); @@ -300,7 +300,8 @@ static RPCHelpMan waitforblock() { return RPCHelpMan{"waitforblock", "\nWaits for a specific new block and returns useful info about it.\n" - "\nReturns the current block on timeout or exit.\n", + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "Block hash to wait for."}, {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, @@ -323,15 +324,22 @@ static RPCHelpMan waitforblock() if (!request.params[1].isNull()) timeout = request.params[1].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning();}); - else - cond_blockchange.wait(lock, [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + const auto deadline{std::chrono::steady_clock::now() + 1ms * timeout}; + while (IsRPCRunning() && block.hash != hash) { + if (timeout) { + auto now{std::chrono::steady_clock::now()}; + if (now >= deadline) break; + const MillisecondsDouble remaining{deadline - now}; + block = miner.waitTipChanged(block.hash, remaining); + } else { + block = miner.waitTipChanged(block.hash); + } } UniValue ret(UniValue::VOBJ); @@ -347,7 +355,8 @@ static RPCHelpMan waitforblockheight() return RPCHelpMan{"waitforblockheight", "\nWaits for (at least) block height and returns the height and hash\n" "of the current tip.\n" - "\nReturns the current block on timeout or exit.\n", + "\nReturns the current block on timeout or exit.\n" + "\nMake sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "Block height to wait for."}, {"timeout", RPCArg::Type::NUM, RPCArg::Default{0}, "Time in milliseconds to wait for a response. 0 indicates no timeout."}, @@ -370,16 +379,25 @@ static RPCHelpMan waitforblockheight() if (!request.params[1].isNull()) timeout = request.params[1].getInt<int>(); + if (timeout < 0) throw JSONRPCError(RPC_MISC_ERROR, "Negative timeout"); - CUpdatedBlock block; - { - WAIT_LOCK(cs_blockchange, lock); - if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning();}); - else - cond_blockchange.wait(lock, [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning(); }); - block = latestblock; + NodeContext& node = EnsureAnyNodeContext(request.context); + Mining& miner = EnsureMining(node); + + auto block{CHECK_NONFATAL(miner.getTip()).value()}; + const auto deadline{std::chrono::steady_clock::now() + 1ms * timeout}; + + while (IsRPCRunning() && block.height < height) { + if (timeout) { + auto now{std::chrono::steady_clock::now()}; + if (now >= deadline) break; + const MillisecondsDouble remaining{deadline - now}; + block = miner.waitTipChanged(block.hash, remaining); + } else { + block = miner.waitTipChanged(block.hash); + } } + UniValue ret(UniValue::VOBJ); ret.pushKV("hash", block.hash.GetHex()); ret.pushKV("height", block.height); @@ -433,6 +451,7 @@ static RPCHelpMan getblockfrompeer() "getblockfrompeer", "Attempt to fetch block from a given peer.\n\n" "We must have the header for this block, e.g. using submitheader.\n" + "The block will not have any undo data which can limit the usage of the block data in a context where the undo data is needed.\n" "Subsequent calls for the same block may cause the response from the previous peer to be ignored.\n" "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n" "When a peer does not respond with a block, we will disconnect.\n" @@ -580,20 +599,32 @@ static RPCHelpMan getblockheader() }; } +void CheckBlockDataAvailability(BlockManager& blockman, const CBlockIndex& blockindex, bool check_for_undo) +{ + AssertLockHeld(cs_main); + uint32_t flag = check_for_undo ? BLOCK_HAVE_UNDO : BLOCK_HAVE_DATA; + if (!(blockindex.nStatus & flag)) { + if (blockman.IsBlockPruned(blockindex)) { + throw JSONRPCError(RPC_MISC_ERROR, strprintf("%s not available (pruned data)", check_for_undo ? "Undo data" : "Block")); + } + if (check_for_undo) { + throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available"); + } + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (not fully downloaded)"); + } +} + static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex& blockindex) { CBlock block; { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/false); } if (!blockman.ReadBlockFromDisk(block, blockindex)) { - // Block not found on disk. This could be because we have the block - // header in our index but not yet have the block or did not accept the - // block. Or if the block was pruned right after we released the lock above. + // Block not found on disk. This shouldn't normally happen unless the block was + // pruned right after we released the lock above. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -606,16 +637,13 @@ static std::vector<uint8_t> GetRawBlockChecked(BlockManager& blockman, const CBl FlatFilePos pos{}; { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/false); pos = blockindex.GetBlockPos(); } if (!blockman.ReadRawBlockFromDisk(data, pos)) { - // Block not found on disk. This could be because we have the block - // header in our index but not yet have the block or did not accept the - // block. Or if the block was pruned right after we released the lock above. + // Block not found on disk. This shouldn't normally happen unless the block was + // pruned right after we released the lock above. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } @@ -631,9 +659,7 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex& bloc { LOCK(cs_main); - if (blockman.IsBlockPruned(blockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); - } + CheckBlockDataAvailability(blockman, blockindex, /*check_for_undo=*/true); } if (!blockman.UndoReadFromDisk(blockUndo, blockindex)) { @@ -656,9 +682,9 @@ const RPCResult getblock_vin{ {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, @@ -740,14 +766,7 @@ static RPCHelpMan getblock() { uint256 hash(ParseHashV(request.params[0], "blockhash")); - int verbosity = 1; - if (!request.params[1].isNull()) { - if (request.params[1].isBool()) { - verbosity = request.params[1].get_bool() ? 1 : 0; - } else { - verbosity = request.params[1].getInt<int>(); - } - } + int verbosity{ParseVerbosity(request.params[1], /*default_verbosity=*/1)}; const CBlockIndex* pblockindex; const CBlockIndex* tip; @@ -786,6 +805,32 @@ static RPCHelpMan getblock() }; } +//! Return height of highest block that has been pruned, or std::nullopt if no blocks have been pruned +std::optional<int> GetPruneHeight(const BlockManager& blockman, const CChain& chain) { + AssertLockHeld(::cs_main); + + // Search for the last block missing block data or undo data. Don't let the + // search consider the genesis block, because the genesis block does not + // have undo data, but should not be considered pruned. + const CBlockIndex* first_block{chain[1]}; + const CBlockIndex* chain_tip{chain.Tip()}; + + // If there are no blocks after the genesis block, or no blocks at all, nothing is pruned. + if (!first_block || !chain_tip) return std::nullopt; + + // If the chain tip is pruned, everything is pruned. + if (!((chain_tip->nStatus & BLOCK_HAVE_MASK) == BLOCK_HAVE_MASK)) return chain_tip->nHeight; + + const auto& first_unpruned{*CHECK_NONFATAL(blockman.GetFirstBlock(*chain_tip, /*status_mask=*/BLOCK_HAVE_MASK, first_block))}; + if (&first_unpruned == first_block) { + // All blocks between first_block and chain_tip have data, so nothing is pruned. + return std::nullopt; + } + + // Block before the first unpruned block is the last pruned block. + return CHECK_NONFATAL(first_unpruned.pprev)->nHeight; +} + static RPCHelpMan pruneblockchain() { return RPCHelpMan{"pruneblockchain", "", @@ -833,13 +878,12 @@ static RPCHelpMan pruneblockchain() } else if (height > chainHeight) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); } else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { - LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks.\n"); + LogDebug(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks.\n"); height = chainHeight - MIN_BLOCKS_TO_KEEP; } PruneBlockFilesManual(active_chainstate, height); - const CBlockIndex& block{*CHECK_NONFATAL(active_chain.Tip())}; - return block.nStatus & BLOCK_HAVE_DATA ? active_chainstate.m_blockman.GetFirstStoredBlock(block)->nHeight - 1 : block.nHeight; + return GetPruneHeight(chainman.m_blockman, active_chain).value_or(-1); }, }; } @@ -1058,9 +1102,9 @@ static RPCHelpMan gettxout() {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, {RPCResult::Type::STR_AMOUNT, "value", "The transaction value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "asm", "Disassembly of the output script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR_HEX, "hex", "The raw output script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, @@ -1249,7 +1293,7 @@ RPCHelpMan getblockchaininfo() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, + {RPCResult::Type::STR, "chain", "current network name (" LIST_CHAIN_NAMES ")"}, {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, @@ -1299,8 +1343,8 @@ RPCHelpMan getblockchaininfo() obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", chainman.m_blockman.IsPruneMode()); if (chainman.m_blockman.IsPruneMode()) { - bool has_tip_data = tip.nStatus & BLOCK_HAVE_DATA; - obj.pushKV("pruneheight", has_tip_data ? chainman.m_blockman.GetFirstStoredBlock(tip)->nHeight : tip.nHeight + 1); + const auto prune_height{GetPruneHeight(chainman.m_blockman, active_chainstate.m_chain)}; + obj.pushKV("pruneheight", prune_height ? prune_height.value() + 1 : 0); const bool automatic_pruning{chainman.m_blockman.GetPruneTarget() != BlockManager::PRUNE_TARGET_MANUAL}; obj.pushKV("automatic_pruning", automatic_pruning); @@ -1552,6 +1596,27 @@ static RPCHelpMan preciousblock() }; } +void InvalidateBlock(ChainstateManager& chainman, const uint256 block_hash) { + BlockValidationState state; + CBlockIndex* pblockindex; + { + LOCK(chainman.GetMutex()); + pblockindex = chainman.m_blockman.LookupBlockIndex(block_hash); + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + } + chainman.ActiveChainstate().InvalidateBlock(state, pblockindex); + + if (state.IsValid()) { + chainman.ActiveChainstate().ActivateBestChain(state); + } + + if (!state.IsValid()) { + throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); + } +} + static RPCHelpMan invalidateblock() { return RPCHelpMan{"invalidateblock", @@ -1566,31 +1631,33 @@ static RPCHelpMan invalidateblock() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + ChainstateManager& chainman = EnsureAnyChainman(request.context); uint256 hash(ParseHashV(request.params[0], "blockhash")); - BlockValidationState state; - ChainstateManager& chainman = EnsureAnyChainman(request.context); - CBlockIndex* pblockindex; + InvalidateBlock(chainman, hash); + + return UniValue::VNULL; +}, + }; +} + +void ReconsiderBlock(ChainstateManager& chainman, uint256 block_hash) { { - LOCK(cs_main); - pblockindex = chainman.m_blockman.LookupBlockIndex(hash); + LOCK(chainman.GetMutex()); + CBlockIndex* pblockindex = chainman.m_blockman.LookupBlockIndex(block_hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - } - chainman.ActiveChainstate().InvalidateBlock(state, pblockindex); - if (state.IsValid()) { - chainman.ActiveChainstate().ActivateBestChain(state); + chainman.ActiveChainstate().ResetBlockFailureFlags(pblockindex); } + BlockValidationState state; + chainman.ActiveChainstate().ActivateBestChain(state); + if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); } - - return UniValue::VNULL; -}, - }; } static RPCHelpMan reconsiderblock() @@ -1611,22 +1678,7 @@ static RPCHelpMan reconsiderblock() ChainstateManager& chainman = EnsureAnyChainman(request.context); uint256 hash(ParseHashV(request.params[0], "blockhash")); - { - LOCK(cs_main); - CBlockIndex* pblockindex = chainman.m_blockman.LookupBlockIndex(hash); - if (!pblockindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - - chainman.ActiveChainstate().ResetBlockFailureFlags(pblockindex); - } - - BlockValidationState state; - chainman.ActiveChainstate().ActivateBestChain(state); - - if (!state.IsValid()) { - throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); - } + ReconsiderBlock(chainman, hash); return UniValue::VNULL; }, @@ -1645,13 +1697,19 @@ static RPCHelpMan getchaintxstats() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM_TIME, "time", "The timestamp for the final block in the window, expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "txcount", "The total number of transactions in the chain up to that point"}, + {RPCResult::Type::NUM, "txcount", /*optional=*/true, + "The total number of transactions in the chain up to that point, if known. " + "It may be unknown when using assumeutxo."}, {RPCResult::Type::STR_HEX, "window_final_block_hash", "The hash of the final block in the window"}, {RPCResult::Type::NUM, "window_final_block_height", "The height of the final block in the window."}, {RPCResult::Type::NUM, "window_block_count", "Size of the window in number of blocks"}, - {RPCResult::Type::NUM, "window_tx_count", /*optional=*/true, "The number of transactions in the window. Only returned if \"window_block_count\" is > 0"}, {RPCResult::Type::NUM, "window_interval", /*optional=*/true, "The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0"}, - {RPCResult::Type::NUM, "txrate", /*optional=*/true, "The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0"}, + {RPCResult::Type::NUM, "window_tx_count", /*optional=*/true, + "The number of transactions in the window. " + "Only returned if \"window_block_count\" is > 0 and if txcount exists for the start and end of the window."}, + {RPCResult::Type::NUM, "txrate", /*optional=*/true, + "The average rate of transactions per second in the window. " + "Only returned if \"window_interval\" is > 0 and if window_tx_count exists."}, }}, RPCExamples{ HelpExampleCli("getchaintxstats", "") @@ -1692,19 +1750,23 @@ static RPCHelpMan getchaintxstats() const CBlockIndex& past_block{*CHECK_NONFATAL(pindex->GetAncestor(pindex->nHeight - blockcount))}; const int64_t nTimeDiff{pindex->GetMedianTimePast() - past_block.GetMedianTimePast()}; - const int nTxDiff = pindex->nChainTx - past_block.nChainTx; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); - ret.pushKV("txcount", (int64_t)pindex->nChainTx); + if (pindex->m_chain_tx_count) { + ret.pushKV("txcount", pindex->m_chain_tx_count); + } ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); ret.pushKV("window_final_block_height", pindex->nHeight); ret.pushKV("window_block_count", blockcount); if (blockcount > 0) { - ret.pushKV("window_tx_count", nTxDiff); ret.pushKV("window_interval", nTimeDiff); - if (nTimeDiff > 0) { - ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + if (pindex->m_chain_tx_count != 0 && past_block.m_chain_tx_count != 0) { + const auto window_tx_count = pindex->m_chain_tx_count - past_block.m_chain_tx_count; + ret.pushKV("window_tx_count", window_tx_count); + if (nTimeDiff > 0) { + ret.pushKV("txrate", double(window_tx_count) / nTimeDiff); + } } } @@ -2126,14 +2188,14 @@ static const auto scan_result_status_some = RPCResult{ static RPCHelpMan scantxoutset() { - // scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S + // raw() descriptor corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S const std::string EXAMPLE_DESCRIPTOR_RAW = "raw(76a91411b366edfc0a8b66feebae5c2e25a7b6a5d1cf3188ac)#fm24fxxy"; return RPCHelpMan{"scantxoutset", "\nScans the unspent transaction output set for entries that match certain output descriptors.\n" "Examples of output descriptors are:\n" - " addr(<address>) Outputs whose scriptPubKey corresponds to the specified address (does not include P2PK)\n" - " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + " addr(<address>) Outputs whose output script corresponds to the specified address (does not include P2PK)\n" + " raw(<hex script>) Outputs whose output script equals the specified hex-encoded bytes\n" " combo(<pubkey>) P2PK, P2PKH, P2WPKH, and P2SH-P2WPKH outputs for the given pubkey\n" " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" @@ -2154,7 +2216,7 @@ static RPCHelpMan scantxoutset() RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "success", "Whether the scan was completed"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs scanned"}, - {RPCResult::Type::NUM, "height", "The current block height (index)"}, + {RPCResult::Type::NUM, "height", "The block height at which the scan was done"}, {RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at the tip of the chain"}, {RPCResult::Type::ARR, "unspents", "", { @@ -2162,11 +2224,13 @@ static RPCHelpMan scantxoutset() { {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, {RPCResult::Type::NUM, "vout", "The vout value"}, - {RPCResult::Type::STR_HEX, "scriptPubKey", "The script key"}, - {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched scriptPubKey"}, + {RPCResult::Type::STR_HEX, "scriptPubKey", "The output script"}, + {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched output script"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, + {RPCResult::Type::STR_HEX, "blockhash", "Blockhash of the unspent transaction output"}, + {RPCResult::Type::NUM, "confirmations", "Number of confirmations of the unspent transaction output when the scan was done"}, }}, }}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, @@ -2256,17 +2320,20 @@ static RPCHelpMan scantxoutset() const COutPoint& outpoint = it.first; const Coin& coin = it.second; const CTxOut& txo = coin.out; + const CBlockIndex& coinb_block{*CHECK_NONFATAL(tip->GetAncestor(coin.nHeight))}; input_txos.push_back(txo); total_in += txo.nValue; UniValue unspent(UniValue::VOBJ); unspent.pushKV("txid", outpoint.hash.GetHex()); - unspent.pushKV("vout", (int32_t)outpoint.n); + unspent.pushKV("vout", outpoint.n); unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); unspent.pushKV("coinbase", coin.IsCoinBase()); - unspent.pushKV("height", (int32_t)coin.nHeight); + unspent.pushKV("height", coin.nHeight); + unspent.pushKV("blockhash", coinb_block.GetBlockHash().GetHex()); + unspent.pushKV("confirmations", tip->nHeight - coin.nHeight + 1); unspents.push_back(std::move(unspent)); } @@ -2601,6 +2668,42 @@ static RPCHelpMan getblockfilter() } /** + * RAII class that disables the network in its constructor and enables it in its + * destructor. + */ +class NetworkDisable +{ + CConnman& m_connman; +public: + NetworkDisable(CConnman& connman) : m_connman(connman) { + m_connman.SetNetworkActive(false); + if (m_connman.GetNetworkActive()) { + throw JSONRPCError(RPC_MISC_ERROR, "Network activity could not be suspended."); + } + }; + ~NetworkDisable() { + m_connman.SetNetworkActive(true); + }; +}; + +/** + * RAII class that temporarily rolls back the local chain in it's constructor + * and rolls it forward again in it's destructor. + */ +class TemporaryRollback +{ + ChainstateManager& m_chainman; + const CBlockIndex& m_invalidate_index; +public: + TemporaryRollback(ChainstateManager& chainman, const CBlockIndex& index) : m_chainman(chainman), m_invalidate_index(index) { + InvalidateBlock(m_chainman, m_invalidate_index.GetBlockHash()); + }; + ~TemporaryRollback() { + ReconsiderBlock(m_chainman, m_invalidate_index.GetBlockHash()); + }; +}; + +/** * Serialize the UTXO set to a file for loading elsewhere. * * @see SnapshotMetadata @@ -2609,9 +2712,20 @@ static RPCHelpMan dumptxoutset() { return RPCHelpMan{ "dumptxoutset", - "Write the serialized UTXO set to a file.", + "Write the serialized UTXO set to a file. This can be used in loadtxoutset afterwards if this snapshot height is supported in the chainparams as well.\n\n" + "Unless the the \"latest\" type is requested, the node will roll back to the requested height and network activity will be suspended during this process. " + "Because of this it is discouraged to interact with the node in any other way during the execution of this call to avoid inconsistent results and race conditions, particularly RPCs that interact with blockstorage.\n\n" + "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", { {"path", RPCArg::Type::STR, RPCArg::Optional::NO, "Path to the output file. If relative, will be prefixed by datadir."}, + {"type", RPCArg::Type::STR, RPCArg::Default(""), "The type of snapshot to create. Can be \"latest\" to create a snapshot of the current UTXO set or \"rollback\" to temporarily roll back the state of the node to a historical block before creating the snapshot of a historical UTXO set. This parameter can be omitted if a separate \"rollback\" named parameter is specified indicating the height or hash of a specific historical block. If \"rollback\" is specified and separate \"rollback\" named parameter is not specified, this will roll back to the latest valid snapshot block that can currently be loaded with loadtxoutset."}, + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", + { + {"rollback", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, + "Height or hash of the block to roll back to before creating the snapshot. Note: The further this number is from the tip, the longer this process will take. Consider setting a higher -rpcclienttimeout value in this case.", + RPCArgOptions{.skip_type_check = true, .type_str = {"", "string or numeric"}}}, + }, + }, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2625,10 +2739,33 @@ static RPCHelpMan dumptxoutset() } }, RPCExamples{ - HelpExampleCli("dumptxoutset", "utxo.dat") + HelpExampleCli("-rpcclienttimeout=0 dumptxoutset", "utxo.dat latest") + + HelpExampleCli("-rpcclienttimeout=0 dumptxoutset", "utxo.dat rollback") + + HelpExampleCli("-rpcclienttimeout=0 -named dumptxoutset", R"(utxo.dat rollback=853456)") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + NodeContext& node = EnsureAnyNodeContext(request.context); + const CBlockIndex* tip{WITH_LOCK(::cs_main, return node.chainman->ActiveChain().Tip())}; + const CBlockIndex* target_index{nullptr}; + const std::string snapshot_type{self.Arg<std::string>("type")}; + const UniValue options{request.params[2].isNull() ? UniValue::VOBJ : request.params[2]}; + if (options.exists("rollback")) { + if (!snapshot_type.empty() && snapshot_type != "rollback") { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid snapshot type \"%s\" specified with rollback option", snapshot_type)); + } + target_index = ParseHashOrHeight(options["rollback"], *node.chainman); + } else if (snapshot_type == "rollback") { + auto snapshot_heights = node.chainman->GetParams().GetAvailableSnapshotHeights(); + CHECK_NONFATAL(snapshot_heights.size() > 0); + auto max_height = std::max_element(snapshot_heights.begin(), snapshot_heights.end()); + target_index = ParseHashOrHeight(*max_height, *node.chainman); + } else if (snapshot_type == "latest") { + target_index = tip; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid snapshot type \"%s\" specified. Please specify \"rollback\" or \"latest\"", snapshot_type)); + } + const ArgsManager& args{EnsureAnyArgsman(request.context)}; const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str())); // Write to a temporary path and then move into `path` on completion @@ -2650,9 +2787,68 @@ static RPCHelpMan dumptxoutset() "Couldn't open file " + temppath.utf8string() + " for writing."); } - NodeContext& node = EnsureAnyNodeContext(request.context); - UniValue result = CreateUTXOSnapshot( - node, node.chainman->ActiveChainstate(), afile, path, temppath); + CConnman& connman = EnsureConnman(node); + const CBlockIndex* invalidate_index{nullptr}; + std::optional<NetworkDisable> disable_network; + std::optional<TemporaryRollback> temporary_rollback; + + // If the user wants to dump the txoutset of the current tip, we don't have + // to roll back at all + if (target_index != tip) { + // If the node is running in pruned mode we ensure all necessary block + // data is available before starting to roll back. + if (node.chainman->m_blockman.IsPruneMode()) { + LOCK(node.chainman->GetMutex()); + const CBlockIndex* current_tip{node.chainman->ActiveChain().Tip()}; + const CBlockIndex* first_block{node.chainman->m_blockman.GetFirstBlock(*current_tip, /*status_mask=*/BLOCK_HAVE_MASK)}; + if (first_block->nHeight > target_index->nHeight) { + throw JSONRPCError(RPC_MISC_ERROR, "Could not roll back to requested height since necessary block data is already pruned."); + } + } + + // Suspend network activity for the duration of the process when we are + // rolling back the chain to get a utxo set from a past height. We do + // this so we don't punish peers that send us that send us data that + // seems wrong in this temporary state. For example a normal new block + // would be classified as a block connecting an invalid block. + // Skip if the network is already disabled because this + // automatically re-enables the network activity at the end of the + // process which may not be what the user wants. + if (connman.GetNetworkActive()) { + disable_network.emplace(connman); + } + + invalidate_index = WITH_LOCK(::cs_main, return node.chainman->ActiveChain().Next(target_index)); + temporary_rollback.emplace(*node.chainman, *invalidate_index); + } + + Chainstate* chainstate; + std::unique_ptr<CCoinsViewCursor> cursor; + CCoinsStats stats; + { + // Lock the chainstate before calling PrepareUtxoSnapshot, to be able + // to get a UTXO database cursor while the chain is pointing at the + // target block. After that, release the lock while calling + // WriteUTXOSnapshot. The cursor will remain valid and be used by + // WriteUTXOSnapshot to write a consistent snapshot even if the + // chainstate changes. + LOCK(node.chainman->GetMutex()); + chainstate = &node.chainman->ActiveChainstate(); + // In case there is any issue with a block being read from disk we need + // to stop here, otherwise the dump could still be created for the wrong + // height. + // The new tip could also not be the target block if we have a stale + // sister block of invalidate_index. This block (or a descendant) would + // be activated as the new tip and we would not get to new_tip_index. + if (target_index != chainstate->m_chain.Tip()) { + LogWarning("dumptxoutset failed to roll back to requested height, reverting to tip.\n"); + throw JSONRPCError(RPC_MISC_ERROR, "Could not roll back to requested height."); + } else { + std::tie(cursor, stats, tip) = PrepareUTXOSnapshot(*chainstate, node.rpc_interruption_point); + } + } + + UniValue result = WriteUTXOSnapshot(*chainstate, cursor.get(), &stats, tip, afile, path, temppath, node.rpc_interruption_point); fs::rename(temppath, path); result.pushKV("path", path.utf8string()); @@ -2661,12 +2857,10 @@ static RPCHelpMan dumptxoutset() }; } -UniValue CreateUTXOSnapshot( - NodeContext& node, +std::tuple<std::unique_ptr<CCoinsViewCursor>, CCoinsStats, const CBlockIndex*> +PrepareUTXOSnapshot( Chainstate& chainstate, - AutoFile& afile, - const fs::path& path, - const fs::path& temppath) + const std::function<void()>& interruption_point) { std::unique_ptr<CCoinsViewCursor> pcursor; std::optional<CCoinsStats> maybe_stats; @@ -2676,7 +2870,7 @@ UniValue CreateUTXOSnapshot( // We need to lock cs_main to ensure that the coinsdb isn't written to // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats // based upon the coinsdb, and (iii) constructing a cursor to the - // coinsdb for use below this block. + // coinsdb for use in WriteUTXOSnapshot. // // Cursors returned by leveldb iterate over snapshots, so the contents // of the pcursor will not be affected by simultaneous writes during @@ -2685,11 +2879,11 @@ UniValue CreateUTXOSnapshot( // See discussion here: // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 // - LOCK(::cs_main); + AssertLockHeld(::cs_main); chainstate.ForceFlushStateToDisk(); - maybe_stats = GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, CoinStatsHashType::HASH_SERIALIZED, node.rpc_interruption_point); + maybe_stats = GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, CoinStatsHashType::HASH_SERIALIZED, interruption_point); if (!maybe_stats) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2698,11 +2892,24 @@ UniValue CreateUTXOSnapshot( tip = CHECK_NONFATAL(chainstate.m_blockman.LookupBlockIndex(maybe_stats->hashBlock)); } + return {std::move(pcursor), *CHECK_NONFATAL(maybe_stats), tip}; +} + +UniValue WriteUTXOSnapshot( + Chainstate& chainstate, + CCoinsViewCursor* pcursor, + CCoinsStats* maybe_stats, + const CBlockIndex* tip, + AutoFile& afile, + const fs::path& path, + const fs::path& temppath, + const std::function<void()>& interruption_point) +{ LOG_TIME_SECONDS(strprintf("writing UTXO snapshot at height %s (%s) to file %s (via %s)", tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), tip->nHeight, maybe_stats->coins_count}; + SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), maybe_stats->coins_count}; afile << metadata; @@ -2733,7 +2940,7 @@ UniValue CreateUTXOSnapshot( pcursor->GetKey(key); last_hash = key.hash; while (pcursor->Valid()) { - if (iter % 5000 == 0) node.rpc_interruption_point(); + if (iter % 5000 == 0) interruption_point(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (key.hash != last_hash) { @@ -2760,10 +2967,21 @@ UniValue CreateUTXOSnapshot( result.pushKV("base_height", tip->nHeight); result.pushKV("path", path.utf8string()); result.pushKV("txoutset_hash", maybe_stats->hashSerialized.ToString()); - result.pushKV("nchaintx", tip->nChainTx); + result.pushKV("nchaintx", tip->m_chain_tx_count); return result; } +UniValue CreateUTXOSnapshot( + node::NodeContext& node, + Chainstate& chainstate, + AutoFile& afile, + const fs::path& path, + const fs::path& tmppath) +{ + auto [cursor, stats, tip]{WITH_LOCK(::cs_main, return PrepareUTXOSnapshot(chainstate, node.rpc_interruption_point))}; + return WriteUTXOSnapshot(chainstate, cursor.get(), &stats, tip, afile, path, tmppath, node.rpc_interruption_point); +} + static RPCHelpMan loadtxoutset() { return RPCHelpMan{ @@ -2798,13 +3016,13 @@ static RPCHelpMan loadtxoutset() } }, RPCExamples{ - HelpExampleCli("loadtxoutset", "utxo.dat") + HelpExampleCli("-rpcclienttimeout=0 loadtxoutset", "utxo.dat") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); - fs::path path{AbsPathForConfigVal(EnsureArgsman(node), fs::u8path(request.params[0].get_str()))}; + const fs::path path{AbsPathForConfigVal(EnsureArgsman(node), fs::u8path(self.Arg<std::string>("path")))}; FILE* file{fsbridge::fopen(path, "rb")}; AutoFile afile{file}; @@ -2821,34 +3039,24 @@ static RPCHelpMan loadtxoutset() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); } - uint256 base_blockhash = metadata.m_base_blockhash; - int base_blockheight = metadata.m_base_blockheight; - if (!chainman.GetParams().AssumeutxoForBlockhash(base_blockhash).has_value()) { - auto available_heights = chainman.GetParams().GetAvailableSnapshotHeights(); - std::string heights_formatted = Join(available_heights, ", ", [&](const auto& i) { return ToString(i); }); - throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to load UTXO snapshot, " - "assumeutxo block hash in snapshot metadata not recognized (hash: %s, height: %s). The following snapshot heights are available: %s.", - base_blockhash.ToString(), - base_blockheight, - heights_formatted)); + auto activation_result{chainman.ActivateSnapshot(afile, metadata, false)}; + if (!activation_result) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to load UTXO snapshot: %s. (%s)", util::ErrorString(activation_result).original, path.utf8string())); } - CBlockIndex* snapshot_start_block = WITH_LOCK(::cs_main, - return chainman.m_blockman.LookupBlockIndex(base_blockhash)); - if (!snapshot_start_block) { - throw JSONRPCError( - RPC_INTERNAL_ERROR, - strprintf("The base block header (%s) must appear in the headers chain. Make sure all headers are syncing, and call this RPC again.", - base_blockhash.ToString())); - } - if (!chainman.ActivateSnapshot(afile, metadata, false)) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to load UTXO snapshot " + fs::PathToString(path)); - } + // Because we can't provide historical blocks during tip or background sync. + // Update local services to reflect we are a limited peer until we are fully sync. + node.connman->RemoveLocalServices(NODE_NETWORK); + // Setting the limited state is usually redundant because the node can always + // provide the last 288 blocks, but it doesn't hurt to set it. + node.connman->AddLocalServices(NODE_NETWORK_LIMITED); + + CBlockIndex& snapshot_index{*CHECK_NONFATAL(*activation_result)}; UniValue result(UniValue::VOBJ); result.pushKV("coins_loaded", metadata.m_coins_count); - result.pushKV("tip_hash", snapshot_start_block->GetBlockHash().ToString()); - result.pushKV("base_height", snapshot_start_block->nHeight); + result.pushKV("tip_hash", snapshot_index.GetBlockHash().ToString()); + result.pushKV("base_height", snapshot_index.nHeight); result.pushKV("path", fs::PathToString(path)); return result; }, |