Merge bitcoin/bitcoin#29612: rpc: Optimize serialization and enhance metadata of dumptxoutset output

542e13b2937356810bda2c41be83c3b1675e2f2f rpc: Enhance metadata of the dumptxoutset output (Fabian Jahr)
4d8e5edbaa94805be41ae4c8aa2f4bf7aaa276fe assumeutxo: Add documentation on dumptxoutset serialization format (Fabian Jahr)
c14ed7f384075330361df636f40121cf25a066d6 assumeutxo: Add test for changed coin size value (Fabian Jahr)
de95953d870c41436de67d56c93259bc66fe1434 rpc: Optimize serialization disk space of dumptxoutset (Fabian Jahr)

Pull request description:

  The second attempt at implementing the `dumptxoutset` space optimization as suggested in #25675. Closes #25675.

  This builds on the work done in #26045, addresses open feedback, adds some further improvements (most importantly usage of compact size), documentation, and an additional test.

  The [original snapshot at height 830,000](https://github.com/bitcoin/bitcoin/pull/29551) came in at 10.82 GB. With this change, the same snapshot is 8.94 GB, a reduction of 17.4%.

  This also enhances the metadata of the output file and adds the following data to allow for better error handling and make future upgrades easier:
  - A newly introduced utxo set magic
  - A version number
  - The network magic
  - The block height

ACKs for top commit:
  achow101:
    ACK 542e13b2937356810bda2c41be83c3b1675e2f2f
  TheCharlatan:
    Re-ACK 542e13b2937356810bda2c41be83c3b1675e2f2f
  theStack:
    ACK 542e13b2937356810bda2c41be83c3b1675e2f2f

Tree-SHA512: 0825d30e5c3c364062db3c6cbca4e3c680e6e6d3e259fa70c0c2b2a7020f24a47406a623582040988d5c7745b08649c31110df4c10656aa25f3f27eb35843d99

1 files changed, 55 insertions, 1 deletions

diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h
index 1160bb55f0..256a4a601d 100644
--- a/src/node/utxo_snapshot.h
+++ b/src/node/utxo_snapshot.h
@@ -6,16 +6,22 @@
 #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
 #define BITCOIN_NODE_UTXO_SNAPSHOT_H
 
+#include <chainparams.h>
+#include <kernel/chainparams.h>
 #include <kernel/cs_main.h>
 #include <serialize.h>
 #include <sync.h>
 #include <uint256.h>
+#include <util/chaintype.h>
 #include <util/fs.h>
 
 #include <cstdint>
 #include <optional>
 #include <string_view>
 
+// UTXO set snapshot magic bytes
+static constexpr std::array<uint8_t, 5> SNAPSHOT_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff};
+
 class Chainstate;
 
 namespace node {
@@ -23,10 +29,14 @@ namespace node {
 //! assumeutxo Chainstate can be constructed.
 class SnapshotMetadata
 {
+    const uint16_t m_version{1};
+    const std::set<uint16_t> m_supported_versions{1};
 public:
     //! The hash of the block that reflects the tip of the chain for the
     //! UTXO set contained in this snapshot.
     uint256 m_base_blockhash;
+    uint32_t m_base_blockheight;
+
 
     //! The number of coins in the UTXO set contained in this snapshot. Used
     //! during snapshot load to estimate progress of UTXO set reconstruction.
@@ -35,11 +45,55 @@ public:
     SnapshotMetadata() { }
     SnapshotMetadata(
         const uint256& base_blockhash,
+        const int base_blockheight,
         uint64_t coins_count) :
             m_base_blockhash(base_blockhash),
+            m_base_blockheight(base_blockheight),
             m_coins_count(coins_count) { }
 
-    SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); }
+    template <typename Stream>
+    inline void Serialize(Stream& s) const {
+        s << SNAPSHOT_MAGIC_BYTES;
+        s << m_version;
+        s << Params().MessageStart();
+        s << m_base_blockheight;
+        s << m_base_blockhash;
+        s << m_coins_count;
+    }
+
+    template <typename Stream>
+    inline void Unserialize(Stream& s) {
+        // Read the snapshot magic bytes
+        std::array<uint8_t, SNAPSHOT_MAGIC_BYTES.size()> snapshot_magic;
+        s >> snapshot_magic;
+        if (snapshot_magic != SNAPSHOT_MAGIC_BYTES) {
+            throw std::ios_base::failure("Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format.");
+        }
+
+        // Read the version
+        uint16_t version;
+        s >> version;
+        if (m_supported_versions.find(version) == m_supported_versions.end()) {
+            throw std::ios_base::failure(strprintf("Version of snapshot %s does not match any of the supported versions.", version));
+        }
+
+        // Read the network magic (pchMessageStart)
+        MessageStartChars message;
+        s >> message;
+        if (!std::equal(message.begin(), message.end(), Params().MessageStart().data())) {
+            auto metadata_network = GetNetworkForMagic(message);
+            if (metadata_network) {
+                std::string network_string{ChainTypeToString(metadata_network.value())};
+                throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, Params().GetChainTypeString()));
+            } else {
+                throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption.");
+            }
+        }
+
+        s >> m_base_blockheight;
+        s >> m_base_blockhash;
+        s >> m_coins_count;
+    }
 };
 
 //! The file in the snapshot chainstate dir which stores the base blockhash. This is