Merge #19806: validation: UTXO snapshot activation

1afc0e4aa1b910991d4f8a77d74e2197f370987c doc: remove potentially confusing ChainstateManager comment (James O'Beirne)
769a1ef9fdc9c372f5bbe91d1961cabd60bc1895 test: Add tests with maleated snapshot data (Fabian Jahr)
4d8de04f32736199e4b41a14a2d29b1a4d0a15d4 tests: add snapshot activation test (James O'Beirne)
31d225274ff1a4b245aea0a69f0e5224b0e64ca2 tests: add deterministic chain generation unittest fixture (James O'Beirne)
6606a4f8c616cf256537c3bfbdade9b43c51b4f5 move-onlyish: break out CreateUTXOSnapshot from dumptxoutset (James O'Beirne)
ad949ba449ff2115e3d22c71f5b6509f11112098 txdb: don't reset during in-memory cache resize (James O'Beirne)
f6e2da5fb7c6406c37612c838c998078ea8d2252 simplify ChainstateManager::SnapshotBlockhash() return semantics (James O'Beirne)
7a6c46b37edb8bfa0085d202aa7e9427d5e4fceb chainparams: add allowed assumeutxo values (James O'Beirne)

Pull request description:

  This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11):

  Parent PR: #15606
  Issue: #15605
  Specification: https://github.com/jamesob/assumeutxo-docs/tree/master/proposal

  ---

  This change proposes logic for activating UTXO snapshots, which is unused at the moment aside from an included unittest. There are a few moveonyish/refactoring commits to allow for halfway decent unittests.

  Basic structure is included for specifying and checking the assumeutxo hash values used to validate activated snapshots. Initially I had specified a few height/hash pairs for mainnet in this change, but because of the security-critical nature of those parameters, I figured it was better to leave their inclusion to a future PR that includes only that change - my intent being that reviewers will be more likely to verify those parameters firsthand in a dedicated PR.

  Aside from that and the snapshot activation logic, there are a few related changes:

  - ~~allow caching the `nChainTx` value in the CCoinsViewDB; this is set during snapshot activation. Because we don't necessarily have access to the full chain at the time of snapshot load, this value is communicated through the snapshot metadata and must be cached within the chainstate to survive restarts.~~
  - break out `CreateUTXOSnapshot()` from dumptxoutset. This is essentially a move-only change to allow the reuse of snapshot creation logic from within unittests.
  - ...and a few other misc. changes that are solely related to unittests.

  The move-onlyish commit is most easily reviewed with `--color-moved=zebra`.

ACKs for top commit:
  fjahr:
    Code review ACK 1afc0e4aa1b910991d4f8a77d74e2197f370987c
  laanwj:
    Code review ACK 1afc0e4aa1b910991d4f8a77d74e2197f370987c

Tree-SHA512: a4e4f0698f00a53ec298b5e8b7ef1c9fdf0185f95139d1b1f63cfdf6cbbd6d17b8c6e51bbf1de2e5f1a946bf49f8466232698ef55acce5a012c80b067da366ea

4 files changed, 255 insertions, 6 deletions

diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp
index b9f3f8c955..790329004c 100644
--- a/src/test/util/setup_common.cpp
+++ b/src/test/util/setup_common.cpp
@@ -199,14 +199,43 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const
     }
 }
 
-TestChain100Setup::TestChain100Setup()
+TestChain100Setup::TestChain100Setup(bool deterministic)
 {
+    m_deterministic = deterministic;
+
+    if (m_deterministic) {
+        SetMockTime(1598887952);
+        constexpr std::array<unsigned char, 32> vchKey = {
+            {
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+            }
+        };
+        coinbaseKey.Set(vchKey.begin(), vchKey.end(), false);
+    } else {
+        coinbaseKey.MakeNewKey(true);
+    }
+
     // Generate a 100-block chain:
-    coinbaseKey.MakeNewKey(true);
+    this->mineBlocks(COINBASE_MATURITY);
+
+    if (m_deterministic) {
+        LOCK(::cs_main);
+        assert(
+            m_node.chainman->ActiveChain().Tip()->GetBlockHash().ToString() ==
+            "49c95db1e470fed04496d801c9d8fbb78155d2c7f855232c918823d2c17d0cf6");
+    }
+}
+
+void TestChain100Setup::mineBlocks(int num_blocks)
+{
     CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
-    for (int i = 0; i < COINBASE_MATURITY; i++) {
+    for (int i = 0; i < num_blocks; i++)
+    {
         std::vector<CMutableTransaction> noTxns;
         CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey);
+        if (m_deterministic) {
+            SetMockTime(GetTime() + 1);
+        }
         m_coinbase_txns.push_back(b.vtx[0]);
     }
 }
@@ -234,6 +263,9 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa
 TestChain100Setup::~TestChain100Setup()
 {
     gArgs.ForceSetArg("-segwitheight", "0");
+    if (m_deterministic) {
+        SetMockTime(0);
+    }
 }
 
 CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const
diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h
index 331c1235cb..4be4763f35 100644
--- a/src/test/util/setup_common.h
+++ b/src/test/util/setup_common.h
@@ -78,7 +78,6 @@ struct BasicTestingSetup {
     explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {});
     ~BasicTestingSetup();
 
-private:
     const fs::path m_path_root;
 };
 
@@ -112,7 +111,7 @@ class CScript;
  * Testing fixture that pre-creates a 100-block REGTEST-mode block chain
  */
 struct TestChain100Setup : public RegTestingSetup {
-    TestChain100Setup();
+    TestChain100Setup(bool deterministic = false);
 
     /**
      * Create a new block with just given transactions, coinbase paying to
@@ -121,12 +120,21 @@ struct TestChain100Setup : public RegTestingSetup {
     CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns,
                                  const CScript& scriptPubKey);
 
+    //! Mine a series of new blocks on the active chain.
+    void mineBlocks(int num_blocks);
+
     ~TestChain100Setup();
 
+    bool m_deterministic;
     std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions
     CKey coinbaseKey; // private/public key needed to spend coinbase transactions
 };
 
+
+struct TestChain100DeterministicSetup : public TestChain100Setup {
+    TestChain100DeterministicSetup() : TestChain100Setup(true) { }
+};
+
 class CTxMemPoolEntry;
 
 struct TestMemPoolEntryHelper
diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp
index 3d8570e27c..94d4277019 100644
--- a/src/test/validation_chainstatemanager_tests.cpp
+++ b/src/test/validation_chainstatemanager_tests.cpp
@@ -4,13 +4,18 @@
 //
 #include <chainparams.h>
 #include <consensus/validation.h>
+#include <node/utxo_snapshot.h>
 #include <random.h>
+#include <rpc/blockchain.h>
 #include <sync.h>
 #include <test/util/setup_common.h>
 #include <uint256.h>
 #include <validation.h>
 #include <validationinterface.h>
 
+#include <tinyformat.h>
+#include <univalue.h>
+
 #include <vector>
 
 #include <boost/test/unit_test.hpp>
@@ -28,6 +33,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
     std::vector<CChainState*> chainstates;
     const CChainParams& chainparams = Params();
 
+    BOOST_CHECK(!manager.SnapshotBlockhash().has_value());
+
     // Create a legacy (IBD) chainstate.
     //
     CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool));
@@ -54,10 +61,17 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
     auto& validated_cs = manager.ValidatedChainstate();
     BOOST_CHECK_EQUAL(&validated_cs, &c1);
 
+    BOOST_CHECK(!manager.SnapshotBlockhash().has_value());
+
     // Create a snapshot-based chainstate.
     //
-    CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool, GetRandHash()));
+    const uint256 snapshot_blockhash = GetRandHash();
+    CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(
+        mempool, snapshot_blockhash));
     chainstates.push_back(&c2);
+
+    BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash);
+
     c2.InitCoinsDB(
         /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false);
     WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23));
@@ -155,4 +169,175 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches)
     BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1);
 }
 
+auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){};
+
+template<typename F = decltype(NoMalleation)>
+static bool
+CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation)
+{
+    // Write out a snapshot to the test's tempdir.
+    //
+    int height;
+    WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight());
+    fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height);
+    FILE* outfile{fsbridge::fopen(snapshot_path, "wb")};
+    CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION};
+
+    UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile);
+    BOOST_TEST_MESSAGE(
+        "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write());
+
+    // Read the written snapshot in and then activate it.
+    //
+    FILE* infile{fsbridge::fopen(snapshot_path, "rb")};
+    CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION};
+    SnapshotMetadata metadata;
+    auto_infile >> metadata;
+
+    malleation(auto_infile, metadata);
+
+    return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true);
+}
+
+//! Test basic snapshot activation.
+BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100DeterministicSetup)
+{
+    ChainstateManager& chainman = *Assert(m_node.chainman);
+
+    size_t initial_size;
+    size_t initial_total_coins{100};
+
+    // Make some initial assertions about the contents of the chainstate.
+    {
+        LOCK(::cs_main);
+        CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip();
+        initial_size = ibd_coinscache.GetCacheSize();
+        size_t total_coins{0};
+
+        for (CTransactionRef& txn : m_coinbase_txns) {
+            COutPoint op{txn->GetHash(), 0};
+            BOOST_CHECK(ibd_coinscache.HaveCoin(op));
+            total_coins++;
+        }
+
+        BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
+        BOOST_CHECK_EQUAL(initial_size, initial_total_coins);
+    }
+
+    // Snapshot should refuse to load at this height.
+    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+    BOOST_CHECK(chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull());
+    BOOST_CHECK_EQUAL(
+        chainman.ActiveChainstate().m_from_snapshot_blockhash,
+        chainman.SnapshotBlockhash().value_or(uint256()));
+
+    // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
+    // be found.
+    mineBlocks(10);
+    initial_size += 10;
+    initial_total_coins += 10;
+
+    // Should not load malleated snapshots
+    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+        m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) {
+            // A UTXO is missing but count is correct
+            metadata.m_coins_count -= 1;
+
+            COutPoint outpoint;
+            Coin coin;
+
+            auto_infile >> outpoint;
+            auto_infile >> coin;
+    }));
+    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+        m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) {
+            // Coins count is larger than coins in file
+            metadata.m_coins_count += 1;
+    }));
+    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
+        m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) {
+            // Coins count is smaller than coins in file
+            metadata.m_coins_count -= 1;
+    }));
+
+    BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+
+    // Ensure our active chain is the snapshot chainstate.
+    BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull());
+    BOOST_CHECK_EQUAL(
+        chainman.ActiveChainstate().m_from_snapshot_blockhash,
+        *chainman.SnapshotBlockhash());
+
+    // To be checked against later when we try loading a subsequent snapshot.
+    uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};
+
+    // Make some assertions about the both chainstates. These checks ensure the
+    // legacy chainstate hasn't changed and that the newly created chainstate
+    // reflects the expected content.
+    {
+        LOCK(::cs_main);
+        int chains_tested{0};
+
+        for (CChainState* chainstate : chainman.GetAll()) {
+            BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
+            CCoinsViewCache& coinscache = chainstate->CoinsTip();
+
+            // Both caches will be empty initially.
+            BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize());
+
+            size_t total_coins{0};
+
+            for (CTransactionRef& txn : m_coinbase_txns) {
+                COutPoint op{txn->GetHash(), 0};
+                BOOST_CHECK(coinscache.HaveCoin(op));
+                total_coins++;
+            }
+
+            BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize());
+            BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
+            chains_tested++;
+        }
+
+        BOOST_CHECK_EQUAL(chains_tested, 2);
+    }
+
+    // Mine some new blocks on top of the activated snapshot chainstate.
+    constexpr size_t new_coins{100};
+    mineBlocks(new_coins);  // Defined in TestChain100Setup.
+
+    {
+        LOCK(::cs_main);
+        size_t coins_in_active{0};
+        size_t coins_in_ibd{0};
+        size_t coins_missing_ibd{0};
+
+        for (CChainState* chainstate : chainman.GetAll()) {
+            BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
+            CCoinsViewCache& coinscache = chainstate->CoinsTip();
+            bool is_ibd = chainman.IsBackgroundIBD(chainstate);
+
+            for (CTransactionRef& txn : m_coinbase_txns) {
+                COutPoint op{txn->GetHash(), 0};
+                if (coinscache.HaveCoin(op)) {
+                    (is_ibd ? coins_in_ibd : coins_in_active)++;
+                } else if (is_ibd) {
+                    coins_missing_ibd++;
+                }
+            }
+        }
+
+        BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins);
+        BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins);
+        BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins);
+    }
+
+    // Snapshot should refuse to load after one has already loaded.
+    BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+
+    // Snapshot blockhash should be unchanged.
+    BOOST_CHECK_EQUAL(
+        chainman.ActiveChainstate().m_from_snapshot_blockhash,
+        loaded_snapshot_blockhash);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp
index 9e37f14921..ecf9453094 100644
--- a/src/test/validation_tests.cpp
+++ b/src/test/validation_tests.cpp
@@ -5,6 +5,7 @@
 #include <chainparams.h>
 #include <net.h>
 #include <signet.h>
+#include <uint256.h>
 #include <validation.h>
 
 #include <test/util/setup_common.h>
@@ -119,4 +120,27 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests)
     BOOST_CHECK(!CheckSignetBlockSolution(block, signet_params->GetConsensus()));
 }
 
+//! Test retrieval of valid assumeutxo values.
+BOOST_AUTO_TEST_CASE(test_assumeutxo)
+{
+    const auto params = CreateChainParams(*m_node.args, CBaseChainParams::REGTEST);
+
+    // These heights don't have assumeutxo configurations associated, per the contents
+    // of chainparams.cpp.
+    std::vector<int> bad_heights{0, 100, 111, 115, 209, 211};
+
+    for (auto empty : bad_heights) {
+        const auto out = ExpectedAssumeutxo(empty, *params);
+        BOOST_CHECK(!out);
+    }
+
+    const auto out110 = *ExpectedAssumeutxo(110, *params);
+    BOOST_CHECK_EQUAL(out110.hash_serialized, uint256S("76fd7334ac7c1baf57ddc0c626f073a655a35d98a4258cd1382c8cc2b8392e10"));
+    BOOST_CHECK_EQUAL(out110.nChainTx, (unsigned int)110);
+
+    const auto out210 = *ExpectedAssumeutxo(210, *params);
+    BOOST_CHECK_EQUAL(out210.hash_serialized, uint256S("9c5ed99ef98544b34f8920b6d1802f72ac28ae6e2bd2bd4c316ff10c230df3f2"));
+    BOOST_CHECK_EQUAL(out210.nChainTx, (unsigned int)210);
+}
+
 BOOST_AUTO_TEST_SUITE_END()