From 5ee22cdafd2562bcb8bf0ae6025e4b53c826382d Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Oct 2021 16:59:18 -0400 Subject: add ChainstateManager.GetSnapshot{BaseHeight,BaseBlock}() For use in later commits. --- src/validation.cpp | 13 +++++++++++++ src/validation.h | 7 +++++++ 2 files changed, 20 insertions(+) (limited to 'src') diff --git a/src/validation.cpp b/src/validation.cpp index f0ffb748dd..70ea99e6df 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5365,3 +5365,16 @@ bool IsBIP30Unspendable(const CBlockIndex& block_index) return (block_index.nHeight==91722 && block_index.GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || (block_index.nHeight==91812 && block_index.GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f")); } + +const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const +{ + const auto blockhash_op = this->SnapshotBlockhash(); + if (!blockhash_op) return nullptr; + return Assert(m_blockman.LookupBlockIndex(*blockhash_op)); +} + +std::optional ChainstateManager::GetSnapshotBaseHeight() const +{ + const CBlockIndex* base = this->GetSnapshotBaseBlock(); + return base ? std::make_optional(base->nHeight) : std::nullopt; +} diff --git a/src/validation.h b/src/validation.h index 7170467b00..6d4959c44d 100644 --- a/src/validation.h +++ b/src/validation.h @@ -869,6 +869,13 @@ private: /** Most recent headers presync progress update, for rate-limiting. */ std::chrono::time_point m_last_presync_update GUARDED_BY(::cs_main) {}; + //! Returns nullptr if no snapshot has been loaded. + const CBlockIndex* GetSnapshotBaseBlock() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Return the height of the base block of the snapshot in use, if one exists, else + //! nullopt. + std::optional GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + public: using Options = kernel::ChainstateManagerOpts; -- cgit v1.2.3 From c29f26b47b8ef978d8689dc0222aa663361ee6cb Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Wed, 2 Feb 2022 14:40:47 -0500 Subject: validation: add CChainState::m_disabled and ChainMan::isUsable and remove m_snapshot_validated. This state can now be inferred by the number of isUsable chainstates. m_disabled is used to signal that a chainstate should no longer be used by validation logic; it is used as a sentinel when background validation completes or if the snapshot chainstate is found to be invalid. isUsable is a convenience method that incorporates m_disabled. --- src/validation.cpp | 19 ++++++++++--------- src/validation.h | 31 ++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/validation.cpp b/src/validation.cpp index 70ea99e6df..9b5875319e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4859,12 +4859,8 @@ std::vector ChainstateManager::GetAll() LOCK(::cs_main); std::vector out; - if (!IsSnapshotValidated() && m_ibd_chainstate) { - out.push_back(m_ibd_chainstate.get()); - } - - if (m_snapshot_chainstate) { - out.push_back(m_snapshot_chainstate.get()); + for (Chainstate* cs : {m_ibd_chainstate.get(), m_snapshot_chainstate.get()}) { + if (this->IsUsable(cs)) out.push_back(cs); } return out; @@ -5263,17 +5259,22 @@ bool ChainstateManager::IsSnapshotActive() const void ChainstateManager::MaybeRebalanceCaches() { AssertLockHeld(::cs_main); - if (m_ibd_chainstate && !m_snapshot_chainstate) { + bool ibd_usable = this->IsUsable(m_ibd_chainstate.get()); + bool snapshot_usable = this->IsUsable(m_snapshot_chainstate.get()); + assert(ibd_usable || snapshot_usable); + + if (ibd_usable && !snapshot_usable) { LogPrintf("[snapshot] allocating all cache to the IBD chainstate\n"); // Allocate everything to the IBD chainstate. m_ibd_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); } - else if (m_snapshot_chainstate && !m_ibd_chainstate) { + else if (snapshot_usable && !ibd_usable) { + // If background validation has completed and snapshot is our active chain... LogPrintf("[snapshot] allocating all cache to the snapshot chainstate\n"); // Allocate everything to the snapshot chainstate. m_snapshot_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); } - else if (m_ibd_chainstate && m_snapshot_chainstate) { + else if (ibd_usable && snapshot_usable) { // If both chainstates exist, determine who needs more cache based on IBD status. // // Note: shrink caches first so that we don't inadvertently overwhelm available memory. diff --git a/src/validation.h b/src/validation.h index 6d4959c44d..2511102b55 100644 --- a/src/validation.h +++ b/src/validation.h @@ -473,6 +473,19 @@ protected: //! Manages the UTXO set, which is a reflection of the contents of `m_chain`. std::unique_ptr m_coins_views; + //! This toggle exists for use when doing background validation for UTXO + //! snapshots. + //! + //! In the expected case, it is set once the background validation chain reaches the + //! same height as the base of the snapshot and its UTXO set is found to hash to + //! the expected assumeutxo value. It signals that we should no longer connect + //! blocks to the background chainstate. When set on the background validation + //! chainstate, it signifies that we have fully validated the snapshot chainstate. + //! + //! In the unlikely case that the snapshot chainstate is found to be invalid, this + //! is set to true on the snapshot chainstate. + bool m_disabled GUARDED_BY(::cs_main) {false}; + public: //! Reference to a BlockManager instance which itself is shared across all //! Chainstate instances. @@ -840,10 +853,6 @@ private: //! that call. Chainstate* m_active_chainstate GUARDED_BY(::cs_main) {nullptr}; - //! If true, the assumed-valid chainstate has been fully validated - //! by the background validation chainstate. - bool m_snapshot_validated GUARDED_BY(::cs_main){false}; - CBlockIndex* m_best_invalid GUARDED_BY(::cs_main){nullptr}; //! Internal helper for ActivateSnapshot(). @@ -876,6 +885,15 @@ private: //! nullopt. std::optional GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! Return true if a chainstate is considered usable. + //! + //! This is false when a background validation chainstate has completed its + //! validation of an assumed-valid chainstate, or when a snapshot + //! chainstate has been found to be invalid. + bool IsUsable(const Chainstate* const cs) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return cs && !cs->m_disabled; + } + public: using Options = kernel::ChainstateManagerOpts; @@ -987,7 +1005,10 @@ public: std::optional SnapshotBlockhash() const; //! Is there a snapshot in use and has it been fully validated? - bool IsSnapshotValidated() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return m_snapshot_validated; } + bool IsSnapshotValidated() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + return m_snapshot_chainstate && m_ibd_chainstate && m_ibd_chainstate->m_disabled; + } /** * Process an incoming block. This only returns after the best known valid -- cgit v1.2.3 From 637a90b973f60555ea4fef4b845ffa7533dcb866 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Wed, 2 Feb 2022 14:44:50 -0500 Subject: add Chainstate::HasCoinsViews() Used in subsequent commits. Also cleans up asserts in coins_views-related convenience methods to be more exact. --- src/validation.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/validation.h b/src/validation.h index 2511102b55..f809c728b6 100644 --- a/src/validation.h +++ b/src/validation.h @@ -553,15 +553,15 @@ public: CCoinsViewCache& CoinsTip() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - assert(m_coins_views->m_cacheview); - return *m_coins_views->m_cacheview.get(); + Assert(m_coins_views); + return *Assert(m_coins_views->m_cacheview); } //! @returns A reference to the on-disk UTXO set database. CCoinsViewDB& CoinsDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - return m_coins_views->m_dbview; + return Assert(m_coins_views)->m_dbview; } //! @returns A pointer to the mempool. @@ -575,12 +575,15 @@ public: CCoinsViewErrorCatcher& CoinsErrorCatcher() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - return m_coins_views->m_catcherview; + return Assert(m_coins_views)->m_catcherview; } //! Destructs all objects related to accessing the UTXO set. void ResetCoinsViews() { m_coins_views.reset(); } + //! Does this chainstate have a UTXO set attached? + bool HasCoinsViews() const { return (bool)m_coins_views; } + //! The cache size of the on-disk coins view. size_t m_coinsdb_cache_size_bytes{0}; -- cgit v1.2.3 From f2a4f3376f1476b38a79a549bd81ba3006225df6 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 3 Nov 2022 14:37:24 -0400 Subject: move-only-ish: init: factor out chainstate initialization Moves chainstate initialization into its own function. This is necessary to later support a more readable way of handling background-validation chainstate cleanup during init, since the chainstate initialization functions may need to be repeated after moving leveldb filesystem content around. This commit isn't strictly necessary, but the alternative is to (ab)use the `while` loop in init.cpp with a `continue` on the basis of a specific ChainstateLoadingError return value from LoadChainstate. Not only is this harder to read, but it can't be unittested. The approach here lets us consolidate background-validation cleanup to LoadChainstate, and therefore exercise it within tests. This commit is most easily reviewed with git diff --color-moved=dimmed-zebra --color-moved-ws=ignore-space-change --- src/node/chainstate.cpp | 78 +++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index ba1024d22e..e9eea90bcb 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -28,38 +28,13 @@ #include namespace node { -ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, - const ChainstateLoadOptions& options) +// Complete initialization of chainstates after the initial call has been made +// to ChainstateManager::InitializeChainstate(). +static ChainstateLoadResult CompleteChainstateInitialization( + ChainstateManager& chainman, + const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); - }; - - if (!chainman.AssumedValidBlock().IsNull()) { - LogPrintf("Assuming ancestors of block %s have valid signatures.\n", chainman.AssumedValidBlock().GetHex()); - } else { - LogPrintf("Validating signatures for all blocks.\n"); - } - LogPrintf("Setting nMinimumChainWork=%s\n", chainman.MinimumChainWork().GetHex()); - if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { - LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); - } - if (chainman.m_blockman.GetPruneTarget() == std::numeric_limits::max()) { - LogPrintf("Block pruning enabled. Use RPC call pruneblockchain(height) to manually prune block and undo files.\n"); - } else if (chainman.m_blockman.GetPruneTarget()) { - LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", chainman.m_blockman.GetPruneTarget() / 1024 / 1024); - } - - LOCK(cs_main); - chainman.m_total_coinstip_cache = cache_sizes.coins; - chainman.m_total_coinsdb_cache = cache_sizes.coins_db; - - // Load the fully validated chainstate. - chainman.InitializeChainstate(options.mempool); - - // Load a chain created from a UTXO snapshot, if any exist. - chainman.DetectSnapshotChainstate(options.mempool); - auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -106,6 +81,10 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } + auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + }; + // Conservative value which is arbitrarily chosen, as it will ultimately be changed // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure // that the sum of the two caches (40%) does not exceed the allowable amount @@ -170,6 +149,43 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {ChainstateLoadStatus::SUCCESS, {}}; } +ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options) +{ + if (!chainman.AssumedValidBlock().IsNull()) { + LogPrintf("Assuming ancestors of block %s have valid signatures.\n", chainman.AssumedValidBlock().GetHex()); + } else { + LogPrintf("Validating signatures for all blocks.\n"); + } + LogPrintf("Setting nMinimumChainWork=%s\n", chainman.MinimumChainWork().GetHex()); + if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { + LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); + } + if (chainman.m_blockman.GetPruneTarget() == std::numeric_limits::max()) { + LogPrintf("Block pruning enabled. Use RPC call pruneblockchain(height) to manually prune block and undo files.\n"); + } else if (chainman.m_blockman.GetPruneTarget()) { + LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", chainman.m_blockman.GetPruneTarget() / 1024 / 1024); + } + + LOCK(cs_main); + + chainman.m_total_coinstip_cache = cache_sizes.coins; + chainman.m_total_coinsdb_cache = cache_sizes.coins_db; + + // Load the fully validated chainstate. + chainman.InitializeChainstate(options.mempool); + + // Load a chain created from a UTXO snapshot, if any exist. + chainman.DetectSnapshotChainstate(options.mempool); + + auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); + if (init_status != ChainstateLoadStatus::SUCCESS) { + return {init_status, init_error}; + } + + return {ChainstateLoadStatus::SUCCESS, {}}; +} + ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { -- cgit v1.2.3 From d96c59cc5cd2f73f1f55c133c52208671fe75ef3 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Apr 2022 10:23:33 -0400 Subject: validation: add ChainMan logic for completing UTXO snapshot validation Trigger completion when a background validation chainstate reaches the same height as a UTXO snapshot, and handle cleaning up the chainstate on subsequent startup. --- src/node/chainstate.cpp | 44 +++++++ src/validation.cpp | 322 +++++++++++++++++++++++++++++++++++++++++++++++- src/validation.h | 58 +++++++++ 3 files changed, 421 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index e9eea90bcb..cd82d8743c 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -85,6 +85,9 @@ static ChainstateLoadResult CompleteChainstateInitialization( return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; + assert(chainman.m_total_coinstip_cache > 0); + assert(chainman.m_total_coinsdb_cache > 0); + // Conservative value which is arbitrarily chosen, as it will ultimately be changed // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure // that the sum of the two caches (40%) does not exceed the allowable amount @@ -183,6 +186,47 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {init_status, init_error}; } + // If a snapshot chainstate was fully validated by a background chainstate during + // the last run, detect it here and clean up the now-unneeded background + // chainstate. + // + // Why is this cleanup done here (on subsequent restart) and not just when the + // snapshot is actually validated? Because this entails unusual + // filesystem operations to move leveldb data directories around, and that seems + // too risky to do in the middle of normal runtime. + auto snapshot_completion = chainman.MaybeCompleteSnapshotValidation(); + + if (snapshot_completion == SnapshotCompletionResult::SKIPPED) { + // do nothing; expected case + } else if (snapshot_completion == SnapshotCompletionResult::SUCCESS) { + LogPrintf("[snapshot] cleaning up unneeded background chainstate, then reinitializing\n"); + if (!chainman.ValidatedSnapshotCleanup()) { + AbortNode("Background chainstate cleanup failed unexpectedly."); + } + + // Because ValidatedSnapshotCleanup() has torn down chainstates with + // ChainstateManager::ResetChainstates(), reinitialize them here without + // duplicating the blockindex work above. + assert(chainman.GetAll().empty()); + assert(!chainman.IsSnapshotActive()); + assert(!chainman.IsSnapshotValidated()); + + chainman.InitializeChainstate(options.mempool); + + // A reload of the block index is required to recompute setBlockIndexCandidates + // for the fully validated chainstate. + chainman.ActiveChainstate().UnloadBlockIndex(); + + auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); + if (init_status != ChainstateLoadStatus::SUCCESS) { + return {init_status, init_error}; + } + } else { + return {ChainstateLoadStatus::FAILURE, _( + "UTXO snapshot failed to validate. " + "Restart to resume normal initial block download, or try loading a different snapshot.")}; + } + return {ChainstateLoadStatus::SUCCESS, {}}; } diff --git a/src/validation.cpp b/src/validation.cpp index 9b5875319e..e4bc6cb10e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2845,6 +2845,14 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, Ticks(time_total), Ticks(time_total) / num_blocks_total); + // If we are the background validation chainstate, check to see if we are done + // validating the snapshot (i.e. our tip has reached the snapshot's base block). + if (this != &m_chainman.ActiveChainstate()) { + // This call may set `m_disabled`, which is referenced immediately afterwards in + // ActivateBestChain, so that we stop connecting blocks past the snapshot base. + m_chainman.MaybeCompleteSnapshotValidation(); + } + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -3067,6 +3075,14 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< // we use m_chainstate_mutex to enforce mutual exclusion so that only one caller may execute this function at a time LOCK(m_chainstate_mutex); + // Belt-and-suspenders check that we aren't attempting to advance the background + // chainstate past the snapshot base block. + if (WITH_LOCK(::cs_main, return m_disabled)) { + LogPrintf("m_disabled is set - this chainstate should not be in operation. " /* Continued */ + "Please report this as a bug. %s\n", PACKAGE_BUGREPORT); + return false; + } + CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); @@ -3117,6 +3133,15 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< assert(trace.pblock && trace.pindex); GetMainSignals().BlockConnected(trace.pblock, trace.pindex); } + + // This will have been toggled in + // ActivateBestChainStep -> ConnectTip -> MaybeCompleteSnapshotValidation, + // if at all, so we should catch it here. + // + // Break this do-while to ensure we don't advance past the base snapshot. + if (m_disabled) { + break; + } } while (!m_chain.Tip() || (starting_tip && CBlockIndexWorkComparator()(m_chain.Tip(), starting_tip))); if (!blocks_connected) return true; @@ -3137,6 +3162,11 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + if (WITH_LOCK(::cs_main, return m_disabled)) { + // Background chainstate has reached the snapshot base block, so exit. + break; + } + // We check shutdown only after giving ActivateBestChainStep a chance to run once so that we // never shutdown before connecting the genesis block during LoadChainTip(). Previously this // caused an assert() failure during shutdown in such cases as the UTXO DB flushing checks @@ -5046,6 +5076,19 @@ static void FlushSnapshotToDisk(CCoinsViewCache& coins_cache, bool snapshot_load coins_cache.Flush(); } +struct StopHashingException : public std::exception +{ + const char* what() const throw() override + { + return "ComputeUTXOStats interrupted by shutdown."; + } +}; + +static void SnapshotUTXOHashBreakpoint() +{ + if (ShutdownRequested()) throw StopHashingException(); +} + bool ChainstateManager::PopulateAndValidateSnapshot( Chainstate& snapshot_chainstate, AutoFile& coins_file, @@ -5169,13 +5212,18 @@ bool ChainstateManager::PopulateAndValidateSnapshot( assert(coins_cache.GetBestBlock() == base_blockhash); - auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; - // As above, okay to immediately release cs_main here since no other context knows // about the snapshot_chainstate. CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); - const std::optional maybe_stats = ComputeUTXOStats(CoinStatsHashType::HASH_SERIALIZED, snapshot_coinsdb, m_blockman, breakpoint_fnc); + std::optional maybe_stats; + + try { + maybe_stats = ComputeUTXOStats( + CoinStatsHashType::HASH_SERIALIZED, snapshot_coinsdb, m_blockman, SnapshotUTXOHashBreakpoint); + } catch (StopHashingException const&) { + return false; + } if (!maybe_stats.has_value()) { LogPrintf("[snapshot] failed to generate coins stats\n"); return false; @@ -5243,6 +5291,149 @@ bool ChainstateManager::PopulateAndValidateSnapshot( return true; } +// Currently, this function holds cs_main for its duration, which could be for +// multiple minutes due to the ComputeUTXOStats call. This hold is necessary +// because we need to avoid advancing the background validation chainstate +// farther than the snapshot base block - and this function is also invoked +// from within ConnectTip, i.e. from within ActivateBestChain, so cs_main is +// held anyway. +// +// Eventually (TODO), we could somehow separate this function's runtime from +// maintenance of the active chain, but that will either require +// +// (i) setting `m_disabled` immediately and ensuring all chainstate accesses go +// through IsUsable() checks, or +// +// (ii) giving each chainstate its own lock instead of using cs_main for everything. +SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation( + std::function shutdown_fnc) +{ + AssertLockHeld(cs_main); + if (m_ibd_chainstate.get() == &this->ActiveChainstate() || + !this->IsUsable(m_snapshot_chainstate.get()) || + !this->IsUsable(m_ibd_chainstate.get()) || + !m_ibd_chainstate->m_chain.Tip()) { + // Nothing to do - this function only applies to the background + // validation chainstate. + return SnapshotCompletionResult::SKIPPED; + } + const int snapshot_tip_height = this->ActiveHeight(); + const int snapshot_base_height = *Assert(this->GetSnapshotBaseHeight()); + const CBlockIndex& index_new = *Assert(m_ibd_chainstate->m_chain.Tip()); + + if (index_new.nHeight < snapshot_base_height) { + // Background IBD not complete yet. + return SnapshotCompletionResult::SKIPPED; + } + + assert(SnapshotBlockhash()); + uint256 snapshot_blockhash = *Assert(SnapshotBlockhash()); + + auto handle_invalid_snapshot = [&]() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + bilingual_str user_error = strprintf(_( + "%s failed to validate the -assumeutxo snapshot state. " + "This indicates a hardware problem, or a bug in the software, or a " + "bad software modification that allowed an invalid snapshot to be " + "loaded. As a result of this, the node will shut down and stop using any " + "state that was built on the snapshot, resetting the chain height " + "from %d to %d. On the next " + "restart, the node will resume syncing from %d " + "without using any snapshot data. " + "Please report this incident to %s, including how you obtained the snapshot. " + "The invalid snapshot chainstate has been left on disk in case it is " + "helpful in diagnosing the issue that caused this error."), + PACKAGE_NAME, snapshot_tip_height, snapshot_base_height, snapshot_base_height, PACKAGE_BUGREPORT + ); + + LogPrintf("[snapshot] !!! %s\n", user_error.original); + LogPrintf("[snapshot] deleting snapshot, reverting to validated chain, and stopping node\n"); + + m_active_chainstate = m_ibd_chainstate.get(); + m_snapshot_chainstate->m_disabled = true; + assert(!this->IsUsable(m_snapshot_chainstate.get())); + assert(this->IsUsable(m_ibd_chainstate.get())); + + m_snapshot_chainstate->InvalidateCoinsDBOnDisk(); + + shutdown_fnc(user_error); + }; + + if (index_new.GetBlockHash() != snapshot_blockhash) { + LogPrintf("[snapshot] supposed base block %s does not match the " /* Continued */ + "snapshot base block %s (height %d). Snapshot is not valid.", + index_new.ToString(), snapshot_blockhash.ToString(), snapshot_base_height); + handle_invalid_snapshot(); + return SnapshotCompletionResult::BASE_BLOCKHASH_MISMATCH; + } + + assert(index_new.nHeight == snapshot_base_height); + + int curr_height = m_ibd_chainstate->m_chain.Height(); + + assert(snapshot_base_height == curr_height); + assert(snapshot_base_height == index_new.nHeight); + assert(this->IsUsable(m_snapshot_chainstate.get())); + assert(this->GetAll().size() == 2); + + CCoinsViewDB& ibd_coins_db = m_ibd_chainstate->CoinsDB(); + m_ibd_chainstate->ForceFlushStateToDisk(); + + auto maybe_au_data = ExpectedAssumeutxo(curr_height, ::Params()); + if (!maybe_au_data) { + LogPrintf("[snapshot] assumeutxo data not found for height " /* Continued */ + "(%d) - refusing to validate snapshot\n", curr_height); + handle_invalid_snapshot(); + return SnapshotCompletionResult::MISSING_CHAINPARAMS; + } + + const AssumeutxoData& au_data = *maybe_au_data; + std::optional maybe_ibd_stats; + LogPrintf("[snapshot] computing UTXO stats for background chainstate to validate " /* Continued */ + "snapshot - this could take a few minutes\n"); + try { + maybe_ibd_stats = ComputeUTXOStats( + CoinStatsHashType::HASH_SERIALIZED, + &ibd_coins_db, + m_blockman, + SnapshotUTXOHashBreakpoint); + } catch (StopHashingException const&) { + return SnapshotCompletionResult::STATS_FAILED; + } + + // XXX note that this function is slow and will hold cs_main for potentially minutes. + if (!maybe_ibd_stats) { + LogPrintf("[snapshot] failed to generate stats for validation coins db\n"); + // While this isn't a problem with the snapshot per se, this condition + // prevents us from validating the snapshot, so we should shut down and let the + // user handle the issue manually. + handle_invalid_snapshot(); + return SnapshotCompletionResult::STATS_FAILED; + } + const auto& ibd_stats = *maybe_ibd_stats; + + // Compare the background validation chainstate's UTXO set hash against the hard-coded + // assumeutxo hash we expect. + // + // TODO: For belt-and-suspenders, we could cache the UTXO set + // hash for the snapshot when it's loaded in its chainstate's leveldb. We could then + // reference that here for an additional check. + if (AssumeutxoHash{ibd_stats.hashSerialized} != au_data.hash_serialized) { + LogPrintf("[snapshot] hash mismatch: actual=%s, expected=%s\n", + ibd_stats.hashSerialized.ToString(), + au_data.hash_serialized.ToString()); + handle_invalid_snapshot(); + return SnapshotCompletionResult::HASH_MISMATCH; + } + + LogPrintf("[snapshot] snapshot beginning at %s has been fully validated\n", + snapshot_blockhash.ToString()); + + m_ibd_chainstate->m_disabled = true; + this->MaybeRebalanceCaches(); + + return SnapshotCompletionResult::SUCCESS; +} + Chainstate& ChainstateManager::ActiveChainstate() const { LOCK(::cs_main); @@ -5367,6 +5558,44 @@ bool IsBIP30Unspendable(const CBlockIndex& block_index) (block_index.nHeight==91812 && block_index.GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f")); } +void Chainstate::InvalidateCoinsDBOnDisk() +{ + AssertLockHeld(::cs_main); + // Should never be called on a non-snapshot chainstate. + assert(m_from_snapshot_blockhash); + auto storage_path_maybe = this->CoinsDB().StoragePath(); + // Should never be called with a non-existent storage path. + assert(storage_path_maybe); + fs::path snapshot_datadir = *storage_path_maybe; + + // Coins views no longer usable. + m_coins_views.reset(); + + auto invalid_path = snapshot_datadir + "_INVALID"; + std::string dbpath = fs::PathToString(snapshot_datadir); + std::string target = fs::PathToString(invalid_path); + LogPrintf("[snapshot] renaming snapshot datadir %s to %s\n", dbpath, target); + + // The invalid snapshot datadir is simply moved and not deleted because we may + // want to do forensics later during issue investigation. The user is instructed + // accordingly in MaybeCompleteSnapshotValidation(). + try { + fs::rename(snapshot_datadir, invalid_path); + } catch (const fs::filesystem_error& e) { + auto src_str = fs::PathToString(snapshot_datadir); + auto dest_str = fs::PathToString(invalid_path); + + LogPrintf("%s: error renaming file '%s' -> '%s': %s\n", + __func__, src_str, dest_str, e.what()); + AbortNode(strprintf( + "Rename of '%s' -> '%s' failed. " + "You should resolve this by manually moving or deleting the invalid " + "snapshot directory %s, otherwise you will encounter the same error again " + "on the next startup.", + src_str, dest_str, src_str)); + } +} + const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const { const auto blockhash_op = this->SnapshotBlockhash(); @@ -5379,3 +5608,90 @@ std::optional ChainstateManager::GetSnapshotBaseHeight() const const CBlockIndex* base = this->GetSnapshotBaseBlock(); return base ? std::make_optional(base->nHeight) : std::nullopt; } + +bool ChainstateManager::ValidatedSnapshotCleanup() +{ + AssertLockHeld(::cs_main); + auto get_storage_path = [](auto& chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) -> std::optional { + if (!(chainstate && chainstate->HasCoinsViews())) { + return {}; + } + return chainstate->CoinsDB().StoragePath(); + }; + std::optional ibd_chainstate_path_maybe = get_storage_path(m_ibd_chainstate); + std::optional snapshot_chainstate_path_maybe = get_storage_path(m_snapshot_chainstate); + + if (!this->IsSnapshotValidated()) { + // No need to clean up. + return false; + } + // If either path doesn't exist, that means at least one of the chainstates + // is in-memory, in which case we can't do on-disk cleanup. You'd better be + // in a unittest! + if (!ibd_chainstate_path_maybe || !snapshot_chainstate_path_maybe) { + LogPrintf("[snapshot] snapshot chainstate cleanup cannot happen with " /* Continued */ + "in-memory chainstates. You are testing, right?\n"); + return false; + } + + const auto& snapshot_chainstate_path = *snapshot_chainstate_path_maybe; + const auto& ibd_chainstate_path = *ibd_chainstate_path_maybe; + + // Since we're going to be moving around the underlying leveldb filesystem content + // for each chainstate, make sure that the chainstates (and their constituent + // CoinsViews members) have been destructed first. + // + // The caller of this method will be responsible for reinitializing chainstates + // if they want to continue operation. + this->ResetChainstates(); + + // No chainstates should be considered usable. + assert(this->GetAll().size() == 0); + + LogPrintf("[snapshot] deleting background chainstate directory (now unnecessary) (%s)\n", + fs::PathToString(ibd_chainstate_path)); + + fs::path tmp_old{ibd_chainstate_path + "_todelete"}; + + auto rename_failed_abort = []( + fs::path p_old, + fs::path p_new, + const fs::filesystem_error& err) { + LogPrintf("%s: error renaming file (%s): %s\n", + __func__, fs::PathToString(p_old), err.what()); + AbortNode(strprintf( + "Rename of '%s' -> '%s' failed. " + "Cannot clean up the background chainstate leveldb directory.", + fs::PathToString(p_old), fs::PathToString(p_new))); + }; + + try { + fs::rename(ibd_chainstate_path, tmp_old); + } catch (const fs::filesystem_error& e) { + rename_failed_abort(ibd_chainstate_path, tmp_old, e); + throw; + } + + LogPrintf("[snapshot] moving snapshot chainstate (%s) to " /* Continued */ + "default chainstate directory (%s)\n", + fs::PathToString(snapshot_chainstate_path), fs::PathToString(ibd_chainstate_path)); + + try { + fs::rename(snapshot_chainstate_path, ibd_chainstate_path); + } catch (const fs::filesystem_error& e) { + rename_failed_abort(snapshot_chainstate_path, ibd_chainstate_path, e); + throw; + } + + if (!DeleteCoinsDBFromDisk(tmp_old, /*is_snapshot=*/false)) { + // No need to AbortNode because once the unneeded bg chainstate data is + // moved, it will not interfere with subsequent initialization. + LogPrintf("Deletion of %s failed. Please remove it manually, as the " /* Continued */ + "directory is now unnecessary.\n", + fs::PathToString(tmp_old)); + } else { + LogPrintf("[snapshot] deleted background chainstate directory (%s)\n", + fs::PathToString(ibd_chainstate_path)); + } + return true; +} diff --git a/src/validation.h b/src/validation.h index f809c728b6..9ba35b538d 100644 --- a/src/validation.h +++ b/src/validation.h @@ -24,6 +24,7 @@ #include #include #include