From 542e13b2937356810bda2c41be83c3b1675e2f2f Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 22 Apr 2024 14:52:40 +0200 Subject: rpc: Enhance metadata of the dumptxoutset output The following data is added: - A newly introduced utxo set magic - A version number - The network magic - The block height --- src/kernel/chainparams.cpp | 30 +++++++++++++++++++ src/kernel/chainparams.h | 3 ++ src/node/utxo_snapshot.h | 56 ++++++++++++++++++++++++++++++++++- src/rpc/blockchain.cpp | 16 ++++++++-- test/functional/feature_assumeutxo.py | 50 +++++++++++++++++++++++++------ test/functional/rpc_dumptxoutset.py | 2 +- 6 files changed, 143 insertions(+), 14 deletions(-) diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 26c261eba2..de142cd2c9 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -542,3 +542,33 @@ std::unique_ptr CChainParams::TestNet() { return std::make_unique(); } + +std::vector CChainParams::GetAvailableSnapshotHeights() const +{ + std::vector heights; + heights.reserve(m_assumeutxo_data.size()); + + for (const auto& data : m_assumeutxo_data) { + heights.emplace_back(data.height); + } + return heights; +} + +std::optional GetNetworkForMagic(MessageStartChars& message) +{ + const auto mainnet_msg = CChainParams::Main()->MessageStart(); + const auto testnet_msg = CChainParams::TestNet()->MessageStart(); + const auto regtest_msg = CChainParams::RegTest({})->MessageStart(); + const auto signet_msg = CChainParams::SigNet({})->MessageStart(); + + if (std::equal(message.begin(), message.end(), mainnet_msg.data())) { + return ChainType::MAIN; + } else if (std::equal(message.begin(), message.end(), testnet_msg.data())) { + return ChainType::TESTNET; + } else if (std::equal(message.begin(), message.end(), regtest_msg.data())) { + return ChainType::REGTEST; + } else if (std::equal(message.begin(), message.end(), signet_msg.data())) { + return ChainType::SIGNET; + } + return std::nullopt; +} diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index 7a5539bc71..f396c1b42c 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -93,6 +93,7 @@ public: const Consensus::Params& GetConsensus() const { return consensus; } const MessageStartChars& MessageStart() const { return pchMessageStart; } uint16_t GetDefaultPort() const { return nDefaultPort; } + std::vector GetAvailableSnapshotHeights() const; const CBlock& GenesisBlock() const { return genesis; } /** Default value for -checkmempool and -checkblockindex argument */ @@ -183,4 +184,6 @@ protected: ChainTxData chainTxData; }; +std::optional GetNetworkForMagic(MessageStartChars& pchMessageStart); + #endif // BITCOIN_KERNEL_CHAINPARAMS_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 1160bb55f0..256a4a601d 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,16 +6,22 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include +#include #include #include #include #include +#include #include #include #include #include +// UTXO set snapshot magic bytes +static constexpr std::array SNAPSHOT_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff}; + class Chainstate; namespace node { @@ -23,10 +29,14 @@ namespace node { //! assumeutxo Chainstate can be constructed. class SnapshotMetadata { + const uint16_t m_version{1}; + const std::set m_supported_versions{1}; public: //! The hash of the block that reflects the tip of the chain for the //! UTXO set contained in this snapshot. uint256 m_base_blockhash; + uint32_t m_base_blockheight; + //! The number of coins in the UTXO set contained in this snapshot. Used //! during snapshot load to estimate progress of UTXO set reconstruction. @@ -35,11 +45,55 @@ public: SnapshotMetadata() { } SnapshotMetadata( const uint256& base_blockhash, + const int base_blockheight, uint64_t coins_count) : m_base_blockhash(base_blockhash), + m_base_blockheight(base_blockheight), m_coins_count(coins_count) { } - SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } + template + inline void Serialize(Stream& s) const { + s << SNAPSHOT_MAGIC_BYTES; + s << m_version; + s << Params().MessageStart(); + s << m_base_blockheight; + s << m_base_blockhash; + s << m_coins_count; + } + + template + inline void Unserialize(Stream& s) { + // Read the snapshot magic bytes + std::array snapshot_magic; + s >> snapshot_magic; + if (snapshot_magic != SNAPSHOT_MAGIC_BYTES) { + throw std::ios_base::failure("Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format."); + } + + // Read the version + uint16_t version; + s >> version; + if (m_supported_versions.find(version) == m_supported_versions.end()) { + throw std::ios_base::failure(strprintf("Version of snapshot %s does not match any of the supported versions.", version)); + } + + // Read the network magic (pchMessageStart) + MessageStartChars message; + s >> message; + if (!std::equal(message.begin(), message.end(), Params().MessageStart().data())) { + auto metadata_network = GetNetworkForMagic(message); + if (metadata_network) { + std::string network_string{ChainTypeToString(metadata_network.value())}; + throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, Params().GetChainTypeString())); + } else { + throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption."); + } + } + + s >> m_base_blockheight; + s >> m_base_blockhash; + s >> m_coins_count; + } }; //! The file in the snapshot chainstate dir which stores the base blockhash. This is diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index cecad55aee..f429826fcb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2691,7 +2691,7 @@ UniValue CreateUTXOSnapshot( tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{tip->GetBlockHash(), maybe_stats->coins_count}; + SnapshotMetadata metadata{tip->GetBlockHash(), tip->nHeight, maybe_stats->coins_count}; afile << metadata; @@ -2804,12 +2804,22 @@ static RPCHelpMan loadtxoutset() } SnapshotMetadata metadata; - afile >> metadata; + try { + afile >> metadata; + } catch (const std::ios_base::failure& e) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); + } uint256 base_blockhash = metadata.m_base_blockhash; + int base_blockheight = metadata.m_base_blockheight; if (!chainman.GetParams().AssumeutxoForBlockhash(base_blockhash).has_value()) { + auto available_heights = chainman.GetParams().GetAvailableSnapshotHeights(); + std::string heights_formatted = Join(available_heights, ", ", [&](const auto& i) { return ToString(i); }); throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to load UTXO snapshot, " - "assumeutxo block hash in snapshot metadata not recognized (%s)", base_blockhash.ToString())); + "assumeutxo block hash in snapshot metadata not recognized (hash: %s, height: %s). The following snapshot heights are available: %s.", + base_blockhash.ToString(), + base_blockheight, + heights_formatted)); } CBlockIndex* snapshot_start_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(base_blockhash)); diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index acc242df40..bb5ee6815a 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -75,26 +75,57 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[1].assert_debug_log([log_msg]): assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot{rpc_details}", self.nodes[1].loadtxoutset, bad_snapshot_path) + self.log.info(" - snapshot file with invalid file magic") + parsing_error_code = -22 + bad_magic = 0xf00f00f000 + with open(bad_snapshot_path, 'wb') as f: + f.write(bad_magic.to_bytes(5, "big") + valid_snapshot_contents[5:]) + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format.", self.nodes[1].loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with unsupported version") + for version in [0, 2]: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:5] + version.to_bytes(2, "little") + valid_snapshot_contents[7:]) + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: Version of snapshot {version} does not match any of the supported versions.", self.nodes[1].loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with mismatching network magic") + invalid_magics = [ + # magic, name, real + [0xf9beb4d9, "main", True], + [0x0b110907, "test", True], + [0x0a03cf40, "signet", True], + [0x00000000, "", False], + [0xffffffff, "", False], + ] + for [magic, name, real] in invalid_magics: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:7] + magic.to_bytes(4, 'big') + valid_snapshot_contents[11:]) + if real: + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: The network of the snapshot ({name}) does not match the network of this node (regtest).", self.nodes[1].loadtxoutset, bad_snapshot_path) + else: + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption.", self.nodes[1].loadtxoutset, bad_snapshot_path) + self.log.info(" - snapshot file referring to a block that is not in the assumeutxo parameters") prev_block_hash = self.nodes[0].getblockhash(SNAPSHOT_BASE_HEIGHT - 1) bogus_block_hash = "0" * 64 # Represents any unknown block hash + # The height is not used for anything critical currently, so we just + # confirm the manipulation in the error message + bogus_height = 1337 for bad_block_hash in [bogus_block_hash, prev_block_hash]: with open(bad_snapshot_path, 'wb') as f: - # block hash of the snapshot base is stored right at the start (first 32 bytes) - f.write(bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[32:]) - error_details = f", assumeutxo block hash in snapshot metadata not recognized ({bad_block_hash})" + f.write(valid_snapshot_contents[:11] + bogus_height.to_bytes(4, "little") + bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[47:]) + error_details = f", assumeutxo block hash in snapshot metadata not recognized (hash: {bad_block_hash}, height: {bogus_height}). The following snapshot heights are available: 110, 299." expected_error(rpc_details=error_details) self.log.info(" - snapshot file with wrong number of coins") - valid_num_coins = int.from_bytes(valid_snapshot_contents[32:32 + 8], "little") + valid_num_coins = int.from_bytes(valid_snapshot_contents[47:47 + 8], "little") for off in [-1, +1]: with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:32]) + f.write(valid_snapshot_contents[:47]) f.write((valid_num_coins + off).to_bytes(8, "little")) - f.write(valid_snapshot_contents[32 + 8:]) + f.write(valid_snapshot_contents[47 + 8:]) expected_error(log_msg=f"bad snapshot - coins left over after deserializing 298 coins" if off == -1 else f"bad snapshot format or truncated snapshot after deserializing 299 coins") - self.log.info(" - snapshot file with alternated but parsable UTXO data results in different hash") cases = [ # (content, offset, wrong_hash, custom_message) @@ -109,9 +140,10 @@ class AssumeutxoTest(BitcoinTestFramework): for content, offset, wrong_hash, custom_message in cases: with open(bad_snapshot_path, "wb") as f: - f.write(valid_snapshot_contents[:(32 + 8 + offset)]) + # Prior to offset: Snapshot magic, snapshot version, network magic, height, hash, coins count + f.write(valid_snapshot_contents[:(5 + 2 + 4 + 4 + 32 + 8 + offset)]) f.write(content) - f.write(valid_snapshot_contents[(32 + 8 + offset + len(content)):]) + f.write(valid_snapshot_contents[(5 + 2 + 4 + 4 + 32 + 8 + offset + len(content)):]) log_msg = custom_message if custom_message is not None else f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}" expected_error(log_msg=log_msg) diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 9853c11b75..c92c8da357 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -43,7 +43,7 @@ class DumptxoutsetTest(BitcoinTestFramework): # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( sha256sum_file(str(expected_path)).hex(), - '3263fc0311ea46415b85513a59ad8fe67806b3cdce66147175ecb9da768d4a99') + '2f775f82811150d310527b5ff773f81fb0fb517e941c543c1f7c4d38fd2717b3') assert_equal( out['txoutset_hash'], 'a0b7baa3bf5ccbd3279728f230d7ca0c44a76e9923fca8f32dbfd08d65ea496a') -- cgit v1.2.3