Merge bitcoin/bitcoin#25667: assumeutxo: snapshot initialization

bf9597606166323158bbf631137b82d41f39334f doc: add note about snapshot chainstate init (James O'Beirne)
e4d799528696c5ede38c257afaffd367917e0de8 test: add testcases for snapshot initialization (James O'Beirne)
cced4e7336d93a2dc88e4a61c49941887766bd72 test: move-only-ish: factor out LoadVerifyActivateChainstate() (James O'Beirne)
51fc9241c08a00f1f407f1534853a5cddbbc0a23 test: allow on-disk coins and block tree dbs in tests (James O'Beirne)
3c361391b8f5971eb3c7b620aa7ad9b437cc515e test: add reset_chainstate parameter for snapshot unittests (James O'Beirne)
00b357c215ed900145bd770525a341ba0ed9c027 validation: add ResetChainstates() (James O'Beirne)
3a29dfbfb2c16a50d854f6f81428a68aa9180509 move-only: test: make snapshot chainstate setup reusable (James O'Beirne)
8153bd9247dad3982d54488bcdb3960470315290 blockmanager: avoid undefined behavior during FlushBlockFile (James O'Beirne)
ad67ff377c2b271cb4683da2fb25fd295557f731 validation: remove snapshot datadirs upon validation failure (James O'Beirne)
34d159033106cc595cfa852695610bfe419c989c add utilities for deleting on-disk leveldb data (James O'Beirne)
252abd1e8bc5cdf4368ad55e827a873240535b28 init: add utxo snapshot detection (James O'Beirne)
f9f1735f139b6a1f1c7fea50717ff90dc4ba2bce validation: rename snapshot chainstate dir (James O'Beirne)
d14bebf100aaaa25c7558eeed8b5c536da99885f db: add StoragePath to CDBWrapper/CCoinsViewDB (James O'Beirne)

Pull request description:

  This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11) (parent PR: https://github.com/bitcoin/bitcoin/pull/15606)

  ---

  Half of the replacement for #24232. The original PR grew larger than expected throughout the review process.

  This change adds the ability to initialize a snapshot-based chainstate during init if one is detected on disk. This is of course unused as of now (aside from in unittests) given that we haven't yet enabled actually loading snapshots.

  Don't be scared! There are some big move-only commits in here.

  Accompanying changes include:

  - moving the snapshot coinsdb directory from being called `chainstate_[base blockhash]` to `chainstate_snapshot`, since we only support one snapshot in use at a time. This simplifies some logic, but it necessitates writing that base blockhash out to a file within the coinsdb dir. See [discussion here](https://github.com/bitcoin/bitcoin/pull/24232#discussion_r832762880).
  - adding a simple fix in `FlushBlockFile()` that avoids a crash when attemping to flush to disk before `LoadBlockIndexDB()` is called, which happens when calling `MaybeRebalanceCaches()` during multiple chainstate init.
  - improving the unittest to allow testing with on-disk chainstates - necessary to test a simulated restart and re-initialization.

ACKs for top commit:
  naumenkogs:
    utACK bf9597606166323158bbf631137b82d41f39334f
  ariard:
    Code Review ACK bf9597606
  ryanofsky:
    Code review ACK bf9597606166323158bbf631137b82d41f39334f. Changes since last review: rebasing, switching from CAutoFile to AutoFile, adding comments, switching from BOOST_CHECK to Assert in test util, using chainman.GetMutex() in tests, destroying one ChainstateManager before creating a new one in tests
  fjahr:
    utACK bf9597606166323158bbf631137b82d41f39334f
  aureleoules:
    ACK bf9597606166323158bbf631137b82d41f39334f

Tree-SHA512: 15ae75caf19f8d12a12d2647c52897904d27b265a7af6b4ae7b858592eeadb8f9da6c2394b6baebec90adc28742c053e3eb506119577dae7c1e722ebb3b7bcc0

4 files changed, 155 insertions, 3 deletions

diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp
index 2d5c91bc5c..04d46f4361 100644
--- a/src/node/blockstorage.cpp
+++ b/src/node/blockstorage.cpp
@@ -524,6 +524,16 @@ void BlockManager::FlushUndoFile(int block_file, bool finalize)
 void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo)
 {
     LOCK(cs_LastBlockFile);
+
+    if (m_blockfile_info.size() < 1) {
+        // Return if we haven't loaded any blockfiles yet. This happens during
+        // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
+        // then calls FlushStateToDisk()), resulting in a call to this function before we
+        // have populated `m_blockfile_info` via LoadBlockIndexDB().
+        return;
+    }
+    assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile);
+
     FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize);
     if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
         AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp
index 3f1d6dd743..26af523491 100644
--- a/src/node/chainstate.cpp
+++ b/src/node/chainstate.cpp
@@ -48,10 +48,15 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
     }
 
     LOCK(cs_main);
-    chainman.InitializeChainstate(options.mempool);
     chainman.m_total_coinstip_cache = cache_sizes.coins;
     chainman.m_total_coinsdb_cache = cache_sizes.coins_db;
 
+    // Load the fully validated chainstate.
+    chainman.InitializeChainstate(options.mempool);
+
+    // Load a chain created from a UTXO snapshot, if any exist.
+    chainman.DetectSnapshotChainstate(options.mempool);
+
     auto& pblocktree{chainman.m_blockman.m_block_tree_db};
     // new CBlockTreeDB tries to delete the existing file, which
     // fails if it's still open from the previous loop. Close it first:
@@ -98,12 +103,20 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")};
     }
 
+    // Conservative value which is arbitrarily chosen, as it will ultimately be changed
+    // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure
+    // that the sum of the two caches (40%) does not exceed the allowable amount
+    // during this temporary initialization state.
+    double init_cache_fraction = 0.2;
+
     // At this point we're either in reindex or we've loaded a useful
     // block tree into BlockIndex()!
 
     for (Chainstate* chainstate : chainman.GetAll()) {
+        LogPrintf("Initializing chainstate %s\n", chainstate->ToString());
+
         chainstate->InitCoinsDB(
-            /*cache_size_bytes=*/cache_sizes.coins_db,
+            /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction,
             /*in_memory=*/options.coins_db_in_memory,
             /*should_wipe=*/options.reindex || options.reindex_chainstate);
 
@@ -125,7 +138,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         }
 
         // The on-disk coinsdb is now in a good state, create the cache
-        chainstate->InitCoinsCache(cache_sizes.coins);
+        chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction);
         assert(chainstate->CanFlushToDisk());
 
         if (!is_coinsview_empty(chainstate)) {
@@ -146,6 +159,11 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         };
     }
 
+    // Now that chainstates are loaded and we're able to flush to
+    // disk, rebalance the coins caches to desired levels based
+    // on the condition of each chainstate.
+    chainman.MaybeRebalanceCaches();
+
     return {ChainstateLoadStatus::SUCCESS, {}};
 }
 
diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp
new file mode 100644
index 0000000000..bab1b75211
--- /dev/null
+++ b/src/node/utxo_snapshot.cpp
@@ -0,0 +1,91 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <node/utxo_snapshot.h>
+
+#include <fs.h>
+#include <logging.h>
+#include <streams.h>
+#include <uint256.h>
+#include <util/system.h>
+#include <validation.h>
+
+#include <cstdio>
+#include <optional>
+
+namespace node {
+
+bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate)
+{
+    AssertLockHeld(::cs_main);
+    assert(snapshot_chainstate.m_from_snapshot_blockhash);
+
+    const std::optional<fs::path> chaindir = snapshot_chainstate.CoinsDB().StoragePath();
+    assert(chaindir); // Sanity check that chainstate isn't in-memory.
+    const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME;
+
+    FILE* file{fsbridge::fopen(write_to, "wb")};
+    AutoFile afile{file};
+    if (afile.IsNull()) {
+        LogPrintf("[snapshot] failed to open base blockhash file for writing: %s\n",
+                  fs::PathToString(write_to));
+        return false;
+    }
+    afile << *snapshot_chainstate.m_from_snapshot_blockhash;
+
+    if (afile.fclose() != 0) {
+        LogPrintf("[snapshot] failed to close base blockhash file %s after writing\n",
+                  fs::PathToString(write_to));
+        return false;
+    }
+    return true;
+}
+
+std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir)
+{
+    if (!fs::exists(chaindir)) {
+        LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */
+            "exists at path %s\n", fs::PathToString(chaindir));
+        return std::nullopt;
+    }
+    const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME;
+    const std::string read_from_str = fs::PathToString(read_from);
+
+    if (!fs::exists(read_from)) {
+        LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */
+            "exists at path %s. Try deleting %s and calling loadtxoutset again?\n",
+            fs::PathToString(chaindir), read_from_str);
+        return std::nullopt;
+    }
+
+    uint256 base_blockhash;
+    FILE* file{fsbridge::fopen(read_from, "rb")};
+    AutoFile afile{file};
+    if (afile.IsNull()) {
+        LogPrintf("[snapshot] failed to open base blockhash file for reading: %s\n",
+            read_from_str);
+        return std::nullopt;
+    }
+    afile >> base_blockhash;
+
+    if (std::fgetc(afile.Get()) != EOF) {
+        LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str);
+    } else if (std::ferror(afile.Get())) {
+        LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str);
+    }
+    return base_blockhash;
+}
+
+std::optional<fs::path> FindSnapshotChainstateDir()
+{
+    fs::path possible_dir =
+        gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX));
+
+    if (fs::exists(possible_dir)) {
+        return possible_dir;
+    }
+    return std::nullopt;
+}
+
+} // namespace node
diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h
index 9dd6f06997..c94521792f 100644
--- a/src/node/utxo_snapshot.h
+++ b/src/node/utxo_snapshot.h
@@ -6,8 +6,14 @@
 #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
 #define BITCOIN_NODE_UTXO_SNAPSHOT_H
 
+#include <fs.h>
 #include <uint256.h>
 #include <serialize.h>
+#include <validation.h>
+
+#include <optional>
+
+extern RecursiveMutex cs_main;
 
 namespace node {
 //! Metadata describing a serialized version of a UTXO set from which an
@@ -33,6 +39,33 @@ public:
 
     SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); }
 };
+
+//! The file in the snapshot chainstate dir which stores the base blockhash. This is
+//! needed to reconstruct snapshot chainstates on init.
+//!
+//! Because we only allow loading a single snapshot at a time, there will only be one
+//! chainstate directory with this filename present within it.
+const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"};
+
+//! Write out the blockhash of the snapshot base block that was used to construct
+//! this chainstate. This value is read in during subsequent initializations and
+//! used to reconstruct snapshot-based chainstates.
+bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate)
+    EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+//! Read the blockhash of the snapshot base block that was used to construct the
+//! chainstate.
+std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir)
+    EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+//! Suffix appended to the chainstate (leveldb) dir when created based upon
+//! a snapshot.
+constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot";
+
+
+//! Return a path to the snapshot-based chainstate dir, if one exists.
+std::optional<fs::path> FindSnapshotChainstateDir();
+
 } // namespace node
 
 #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H