diff options
Diffstat (limited to 'src')
76 files changed, 2065 insertions, 622 deletions
diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5dc20d4fab..d66f5bf53a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -300,6 +300,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/netbase_dns_lookup.cpp \ test/fuzz/node_eviction.cpp \ test/fuzz/p2p_transport_serialization.cpp \ + test/fuzz/package_eval.cpp \ test/fuzz/parse_hd_keypath.cpp \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ diff --git a/src/bench/disconnected_transactions.cpp b/src/bench/disconnected_transactions.cpp index d6f1590950..0a7344b248 100644 --- a/src/bench/disconnected_transactions.cpp +++ b/src/bench/disconnected_transactions.cpp @@ -98,7 +98,7 @@ static void AddAndRemoveDisconnectedBlockTransactionsAll(benchmark::Bench& bench const auto chains{CreateBlocks(/*num_not_shared=*/1)}; assert(chains.num_shared == BLOCK_VTX_COUNT - 1); - bench.minEpochIterations(10).run([&]() NO_THREAD_SAFETY_ANALYSIS { + bench.minEpochIterations(10).run([&]() { Reorg(chains); }); } @@ -109,7 +109,7 @@ static void AddAndRemoveDisconnectedBlockTransactions90(benchmark::Bench& bench) const auto chains{CreateBlocks(/*num_not_shared=*/BLOCK_VTX_COUNT_10PERCENT)}; assert(chains.num_shared == BLOCK_VTX_COUNT - BLOCK_VTX_COUNT_10PERCENT); - bench.minEpochIterations(10).run([&]() NO_THREAD_SAFETY_ANALYSIS { + bench.minEpochIterations(10).run([&]() { Reorg(chains); }); } @@ -120,7 +120,7 @@ static void AddAndRemoveDisconnectedBlockTransactions10(benchmark::Bench& bench) const auto chains{CreateBlocks(/*num_not_shared=*/BLOCK_VTX_COUNT - BLOCK_VTX_COUNT_10PERCENT)}; assert(chains.num_shared == BLOCK_VTX_COUNT_10PERCENT); - bench.minEpochIterations(10).run([&]() NO_THREAD_SAFETY_ANALYSIS { + bench.minEpochIterations(10).run([&]() { Reorg(chains); }); } diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index 5e5bc76fd2..160534b63c 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -70,7 +70,7 @@ void generateFakeBlock(const CChainParams& params, // notify wallet const auto& pindex = WITH_LOCK(::cs_main, return context.chainman->ActiveChain().Tip()); - wallet.blockConnected(kernel::MakeBlockInfo(pindex, &block)); + wallet.blockConnected(ChainstateRole::NORMAL, kernel::MakeBlockInfo(pindex, &block)); } struct PreSelectInputs { diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 985a81f522..dd3824fb1c 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -16,9 +16,6 @@ #include <util/golombrice.h> #include <util/string.h> -/// SerType used to serialize parameters in GCS filter encoding. -static constexpr int GCS_SER_TYPE = SER_NETWORK; - /// Protocol version used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_VERSION = 0; @@ -52,7 +49,7 @@ GCSFilter::GCSFilter(const Params& params) GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { - SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; + SpanReader stream{GCS_SER_VERSION, m_encoded}; uint64_t N = ReadCompactSize(stream); m_N = static_cast<uint32_t>(N); @@ -84,7 +81,7 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) } m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_params.m_M); - CVectorWriter stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); + CVectorWriter stream(GCS_SER_VERSION, m_encoded, 0); WriteCompactSize(stream, m_N); @@ -106,7 +103,7 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const { - SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; + SpanReader stream{GCS_SER_VERSION, m_encoded}; // Seek forward by size of N uint64_t N = ReadCompactSize(stream); diff --git a/src/chain.h b/src/chain.h index 7806720ce9..78b06719f4 100644 --- a/src/chain.h +++ b/src/chain.h @@ -276,6 +276,12 @@ public: * * Does not imply the transactions are consensus-valid (ConnectTip might fail) * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true) + * + * Note that this will be true for the snapshot base block, if one is loaded (and + * all subsequent assumed-valid blocks) since its nChainTx value will have been set + * manually based on the related AssumeutxoData entry. + * + * TODO: potentially change the name of this based on the fact above. */ bool HaveTxsDownloaded() const { return nChainTx != 0; } diff --git a/src/hash.h b/src/hash.h index f2b627ff4f..d355b703ff 100644 --- a/src/hash.h +++ b/src/hash.h @@ -149,13 +149,11 @@ public: class CHashWriter : public HashWriter { private: - const int nType; const int nVersion; public: - CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} + CHashWriter(int nVersionIn) : nVersion{nVersionIn} {} - int GetType() const { return nType; } int GetVersion() const { return nVersion; } template<typename T> @@ -223,15 +221,6 @@ public: } }; -/** Compute the 256-bit hash of an object's serialization. */ -template<typename T> -uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) -{ - CHashWriter ss(nType, nVersion); - ss << obj; - return ss.GetHash(); -} - /** Single-SHA256 a 32-byte input (represented as uint256). */ [[nodiscard]] uint256 SHA256Uint256(const uint256& input); diff --git a/src/index/base.cpp b/src/index/base.cpp index f18205a76f..8474d01c41 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -79,9 +79,15 @@ BaseIndex::~BaseIndex() bool BaseIndex::Init() { + AssertLockNotHeld(cs_main); + + // May need reset if index is being restarted. + m_interrupt.reset(); + // m_chainstate member gives indexing code access to node internals. It is // removed in followup https://github.com/bitcoin/bitcoin/pull/24230 - m_chainstate = &m_chain->context()->chainman->ActiveChainstate(); + m_chainstate = WITH_LOCK(::cs_main, + return &m_chain->context()->chainman->GetChainstateForIndexing()); // Register to validation interface before setting the 'm_synced' flag, so that // callbacks are not missed once m_synced is true. RegisterValidationInterface(this); @@ -92,7 +98,8 @@ bool BaseIndex::Init() } LOCK(cs_main); - CChain& active_chain = m_chainstate->m_chain; + CChain& index_chain = m_chainstate->m_chain; + if (locator.IsNull()) { SetBestBlockIndex(nullptr); } else { @@ -114,7 +121,7 @@ bool BaseIndex::Init() // Note: this will latch to true immediately if the user starts up with an empty // datadir and an index enabled. If this is the case, indexation will happen solely // via `BlockConnected` signals until, possibly, the next restart. - m_synced = start_block == active_chain.Tip(); + m_synced = start_block == index_chain.Tip(); m_init = true; return true; } @@ -143,6 +150,8 @@ void BaseIndex::ThreadSync() std::chrono::steady_clock::time_point last_locator_write_time{0s}; while (true) { if (m_interrupt) { + LogPrintf("%s: m_interrupt set; exiting ThreadSync\n", GetName()); + SetBestBlockIndex(pindex); // No need to handle errors in Commit. If it fails, the error will be already be // logged. The best way to recover is to continue, as index cannot be corrupted by @@ -250,8 +259,19 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti return true; } -void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) +void BaseIndex::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) { + // Ignore events from the assumed-valid chain; we will process its blocks + // (sequentially) after it is fully verified by the background chainstate. This + // is to avoid any out-of-order indexing. + // + // TODO at some point we could parameterize whether a particular index can be + // built out of order, but for now just do the conservative simple thing. + if (role == ChainstateRole::ASSUMEDVALID) { + return; + } + + // Ignore BlockConnected signals until we have fully indexed the chain. if (!m_synced) { return; } @@ -296,8 +316,14 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const } } -void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) +void BaseIndex::ChainStateFlushed(ChainstateRole role, const CBlockLocator& locator) { + // Ignore events from the assumed-valid chain; we will process its blocks + // (sequentially) after it is fully verified by the background chainstate. + if (role == ChainstateRole::ASSUMEDVALID) { + return; + } + if (!m_synced) { return; } diff --git a/src/index/base.h b/src/index/base.h index 9b2a41dc92..154061fb19 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -15,6 +15,7 @@ class CBlock; class CBlockIndex; class Chainstate; +class ChainstateManager; namespace interfaces { class Chain; } // namespace interfaces @@ -30,6 +31,11 @@ struct IndexSummary { * Base class for indices of blockchain data. This implements * CValidationInterface and ensures blocks are indexed sequentially according * to their position in the active chain. + * + * In the presence of multiple chainstates (i.e. if a UTXO snapshot is loaded), + * only the background "IBD" chainstate will be indexed to avoid building the + * index out of order. When the background chainstate completes validation, the + * index will be reinitialized and indexing will continue. */ class BaseIndex : public CValidationInterface { @@ -102,9 +108,9 @@ protected: Chainstate* m_chainstate{nullptr}; const std::string m_name; - void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override; + void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override; - void ChainStateFlushed(const CBlockLocator& locator) override; + void ChainStateFlushed(ChainstateRole role, const CBlockLocator& locator) override; /// Initialize internal state from the database and block index. [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockKey>& block) { return true; } @@ -122,9 +128,6 @@ protected: virtual DB& GetDB() const = 0; - /// Get the name of the index for display in logs. - const std::string& GetName() const LIFETIMEBOUND { return m_name; } - /// Update the internal best block index as well as the prune lock. void SetBestBlockIndex(const CBlockIndex* block); @@ -133,6 +136,9 @@ public: /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); + /// Get the name of the index for display in logs. + const std::string& GetName() const LIFETIMEBOUND { return m_name; } + /// Blocks the current thread until the index is caught up to the current /// state of the block chain. This only blocks if the index has gotten in /// sync once and only needs to process blocks in the ValidationInterface diff --git a/src/init.cpp b/src/init.cpp index 6dd3d5970b..a0b4425898 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -498,6 +498,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2pacceptincoming", strprintf("Whether to accept inbound I2P connections (default: %i). Ignored if -i2psam is not set. Listening for inbound I2P connections is done through the SAM proxy, not by binding to a local address and port.", DEFAULT_I2P_ACCEPT_INCOMING), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onlynet=<net>", "Make automatic outbound connections only to network <net> (" + Join(GetNetworkNames(), ", ") + "). Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-v2transport", strprintf("Support v2 transport (default: %u)", DEFAULT_V2_TRANSPORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); @@ -893,6 +894,11 @@ bool AppInitParameterInteraction(const ArgsManager& args) } } + // Signal NODE_P2P_V2 if BIP324 v2 transport is enabled. + if (args.GetBoolArg("-v2transport", DEFAULT_V2_TRANSPORT)) { + nLocalServices = ServiceFlags(nLocalServices | NODE_P2P_V2); + } + // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled. if (args.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) { if (g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { @@ -1478,6 +1484,25 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.chainman = std::make_unique<ChainstateManager>(node.kernel->interrupt, chainman_opts, blockman_opts); ChainstateManager& chainman = *node.chainman; + // This is defined and set here instead of inline in validation.h to avoid a hard + // dependency between validation and index/base, since the latter is not in + // libbitcoinkernel. + chainman.restart_indexes = [&node]() { + LogPrintf("[snapshot] restarting indexes\n"); + + // Drain the validation interface queue to ensure that the old indexes + // don't have any pending work. + SyncWithValidationInterfaceQueue(); + + for (auto* index : node.indexes) { + index->Interrupt(); + index->Stop(); + if (!(index->Init() && index->StartBackgroundSync())) { + LogPrintf("[snapshot] WARNING failed to restart index %s on snapshot chain\n", index->GetName()); + } + } + }; + node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); options.reindex = node::fReindex; @@ -1906,18 +1931,19 @@ bool StartIndexBackgroundSync(NodeContext& node) // indexes_start_block='nullptr' means "start from height 0". std::optional<const CBlockIndex*> indexes_start_block; std::string older_index_name; - ChainstateManager& chainman = *Assert(node.chainman); + const Chainstate& chainstate = WITH_LOCK(::cs_main, return chainman.GetChainstateForIndexing()); + const CChain& index_chain = chainstate.m_chain; + for (auto index : node.indexes) { const IndexSummary& summary = index->GetSummary(); if (summary.synced) continue; // Get the last common block between the index best block and the active chain LOCK(::cs_main); - const CChain& active_chain = chainman.ActiveChain(); const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(summary.best_block_hash); - if (!active_chain.Contains(pindex)) { - pindex = active_chain.FindFork(pindex); + if (!index_chain.Contains(pindex)) { + pindex = index_chain.FindFork(pindex); } if (!indexes_start_block || !pindex || pindex->nHeight < indexes_start_block.value()->nHeight) { @@ -1932,7 +1958,7 @@ bool StartIndexBackgroundSync(NodeContext& node) LOCK(::cs_main); const CBlockIndex* start_block = *indexes_start_block; if (!start_block) start_block = chainman.ActiveChain().Genesis(); - if (!chainman.m_blockman.CheckBlockDataAvailability(*chainman.ActiveChain().Tip(), *Assert(start_block))) { + if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(start_block))) { return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), older_index_name)); } } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index b5243725ad..dea868f844 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -27,6 +27,7 @@ class Coin; class uint256; enum class MemPoolRemovalReason; enum class RBFTransactionState; +enum class ChainstateRole; struct bilingual_str; struct CBlockLocator; struct FeeCalculation; @@ -310,10 +311,10 @@ public: virtual ~Notifications() {} virtual void transactionAddedToMempool(const CTransactionRef& tx) {} virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} - virtual void blockConnected(const BlockInfo& block) {} + virtual void blockConnected(ChainstateRole role, const BlockInfo& block) {} virtual void blockDisconnected(const BlockInfo& block) {} virtual void updatedBlockTip() {} - virtual void chainStateFlushed(const CBlockLocator& locator) {} + virtual void chainStateFlushed(ChainstateRole role, const CBlockLocator& locator) {} }; //! Register handler for notifications. diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 1c877866d0..318c956b38 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -4,6 +4,7 @@ #include <chain.h> #include <interfaces/chain.h> +#include <kernel/chain.h> #include <sync.h> #include <uint256.h> @@ -25,3 +26,13 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data return info; } } // namespace kernel + +std::ostream& operator<<(std::ostream& os, const ChainstateRole& role) { + switch(role) { + case ChainstateRole::NORMAL: os << "normal"; break; + case ChainstateRole::ASSUMEDVALID: os << "assumedvalid"; break; + case ChainstateRole::BACKGROUND: os << "background"; break; + default: os.setstate(std::ios_base::failbit); + } + return os; +} diff --git a/src/kernel/chain.h b/src/kernel/chain.h index f0750f8266..feba24a557 100644 --- a/src/kernel/chain.h +++ b/src/kernel/chain.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_KERNEL_CHAIN_H #define BITCOIN_KERNEL_CHAIN_H +#include<iostream> + class CBlock; class CBlockIndex; namespace interfaces { @@ -14,6 +16,24 @@ struct BlockInfo; namespace kernel { //! Return data from block index. interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* block_index, const CBlock* data = nullptr); + } // namespace kernel +//! This enum describes the various roles a specific Chainstate instance can take. +//! Other parts of the system sometimes need to vary in behavior depending on the +//! existence of a background validation chainstate, e.g. when building indexes. +enum class ChainstateRole { + // Single chainstate in use, "normal" IBD mode. + NORMAL, + + // Doing IBD-style validation in the background. Implies use of an assumed-valid + // chainstate. + BACKGROUND, + + // Active assumed-valid chainstate. Implies use of a background IBD chainstate. + ASSUMEDVALID, +}; + +std::ostream& operator<<(std::ostream& os, const ChainstateRole& role); + #endif // BITCOIN_KERNEL_CHAIN_H diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 7e69c097a6..5e893a3f58 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -172,8 +172,8 @@ public: } }; - m_assumeutxo_data = MapAssumeutxo{ - // TODO to be specified in a future patch. + m_assumeutxo_data = { + // TODO to be specified in a future patch. }; chainTxData = ChainTxData{ @@ -266,8 +266,13 @@ public: } }; - m_assumeutxo_data = MapAssumeutxo{ - // TODO to be specified in a future patch. + m_assumeutxo_data = { + { + .height = 2'500'000, + .hash_serialized = AssumeutxoHash{uint256S("0x2a8fdefef3bf75fa00540ccaaaba4b5281bea94229327bdb0f7416ef1e7a645c")}, + .nChainTx = 66484552, + .blockhash = uint256S("0x0000000000000093bcb68c03a9a168ae252572d348a2eaeba2cdf9231d73206f") + } }; chainTxData = ChainTxData{ @@ -370,6 +375,15 @@ public: vFixedSeeds.clear(); + m_assumeutxo_data = { + { + .height = 160'000, + .hash_serialized = AssumeutxoHash{uint256S("0x5225141cb62dee63ab3be95f9b03d60801f264010b1816d4bd00618b2736e7be")}, + .nChainTx = 2289496, + .blockhash = uint256S("0x0000003ca3c99aff040f2563c2ad8f8ec88bd0fd6b8f0895cfaf1ef90353a62c") + } + }; + base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); base58Prefixes[SECRET_KEY] = std::vector<unsigned char>(1,239); @@ -477,14 +491,19 @@ public: } }; - m_assumeutxo_data = MapAssumeutxo{ + m_assumeutxo_data = { { - 110, - {AssumeutxoHash{uint256S("0x1ebbf5850204c0bdb15bf030f47c7fe91d45c44c712697e4509ba67adb01c618")}, 110}, + .height = 110, + .hash_serialized = AssumeutxoHash{uint256S("0x1ebbf5850204c0bdb15bf030f47c7fe91d45c44c712697e4509ba67adb01c618")}, + .nChainTx = 110, + .blockhash = uint256S("0x696e92821f65549c7ee134edceeeeaaa4105647a3c4fd9f298c0aec0ab50425c") }, { - 200, - {AssumeutxoHash{uint256S("0x51c8d11d8b5c1de51543c579736e786aa2736206d1e11e627568029ce092cf62")}, 200}, + // For use by test/functional/feature_assumeutxo.py + .height = 299, + .hash_serialized = AssumeutxoHash{uint256S("0xef45ccdca5898b6c2145e4581d2b88c56564dd389e4bd75a1aaf6961d3edd3c0")}, + .nChainTx = 300, + .blockhash = uint256S("0x7e0517ef3ea6ecbed9117858e42eedc8eb39e8698a38dcbd1b3962a283233f4c") }, }; diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index ec1697493c..7a5539bc71 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -12,6 +12,7 @@ #include <uint256.h> #include <util/chaintype.h> #include <util/hash_type.h> +#include <util/vector.h> #include <cstdint> #include <iterator> @@ -44,17 +45,21 @@ struct AssumeutxoHash : public BaseHash<uint256> { * as valid. */ struct AssumeutxoData { + int height; + //! The expected hash of the deserialized UTXO set. - const AssumeutxoHash hash_serialized; + AssumeutxoHash hash_serialized; //! Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex(). //! //! We need to hardcode the value here because this is computed cumulatively using block data, //! which we do not necessarily have at the time of snapshot load. - const unsigned int nChainTx; -}; + unsigned int nChainTx; -using MapAssumeutxo = std::map<int, const AssumeutxoData>; + //! The hash of the base block for this snapshot. Used to refer to assumeutxo data + //! prior to having a loaded blockindex. + uint256 blockhash; +}; /** * Holds various statistics on transactions within a chain. Used to estimate @@ -114,9 +119,14 @@ public: const std::vector<uint8_t>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } - //! Get allowed assumeutxo configuration. - //! @see ChainstateManager - const MapAssumeutxo& Assumeutxo() const { return m_assumeutxo_data; } + std::optional<AssumeutxoData> AssumeutxoForHeight(int height) const + { + return FindFirst(m_assumeutxo_data, [&](const auto& d) { return d.height == height; }); + } + std::optional<AssumeutxoData> AssumeutxoForBlockhash(const uint256& blockhash) const + { + return FindFirst(m_assumeutxo_data, [&](const auto& d) { return d.blockhash == blockhash; }); + } const ChainTxData& TxData() const { return chainTxData; } @@ -169,7 +179,7 @@ protected: bool fDefaultConsistencyChecks; bool m_is_mockable_chain; CCheckpointData checkpointData; - MapAssumeutxo m_assumeutxo_data; + std::vector<AssumeutxoData> m_assumeutxo_data; ChainTxData chainTxData; }; diff --git a/src/key.cpp b/src/key.cpp index efaea5b1b3..0f283ca3e3 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -159,21 +159,21 @@ bool CKey::Check(const unsigned char *vch) { } void CKey::MakeNewKey(bool fCompressedIn) { + MakeKeyData(); do { - GetStrongRandBytes(keydata); - } while (!Check(keydata.data())); - fValid = true; + GetStrongRandBytes(*keydata); + } while (!Check(keydata->data())); fCompressed = fCompressedIn; } bool CKey::Negate() { - assert(fValid); - return secp256k1_ec_seckey_negate(secp256k1_context_sign, keydata.data()); + assert(keydata); + return secp256k1_ec_seckey_negate(secp256k1_context_sign, keydata->data()); } CPrivKey CKey::GetPrivKey() const { - assert(fValid); + assert(keydata); CPrivKey seckey; int ret; size_t seckeylen; @@ -186,7 +186,7 @@ CPrivKey CKey::GetPrivKey() const { } CPubKey CKey::GetPubKey() const { - assert(fValid); + assert(keydata); secp256k1_pubkey pubkey; size_t clen = CPubKey::SIZE; CPubKey result; @@ -212,7 +212,7 @@ bool SigHasLowR(const secp256k1_ecdsa_signature* sig) } bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, bool grind, uint32_t test_case) const { - if (!fValid) + if (!keydata) return false; vchSig.resize(CPubKey::SIGNATURE_SIZE); size_t nSigLen = CPubKey::SIGNATURE_SIZE; @@ -253,7 +253,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { } bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) const { - if (!fValid) + if (!keydata) return false; vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; @@ -301,10 +301,12 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 } bool CKey::Load(const CPrivKey &seckey, const CPubKey &vchPubKey, bool fSkipCheck=false) { - if (!ec_seckey_import_der(secp256k1_context_sign, (unsigned char*)begin(), seckey.data(), seckey.size())) + MakeKeyData(); + if (!ec_seckey_import_der(secp256k1_context_sign, (unsigned char*)begin(), seckey.data(), seckey.size())) { + ClearKeyData(); return false; + } fCompressed = vchPubKey.IsCompressed(); - fValid = true; if (fSkipCheck) return true; @@ -325,22 +327,21 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const BIP32Hash(cc, nChild, 0, begin(), vout.data()); } memcpy(ccChild.begin(), vout.data()+32, 32); - memcpy((unsigned char*)keyChild.begin(), begin(), 32); + keyChild.Set(begin(), begin() + 32, true); bool ret = secp256k1_ec_seckey_tweak_add(secp256k1_context_sign, (unsigned char*)keyChild.begin(), vout.data()); - keyChild.fCompressed = true; - keyChild.fValid = ret; + if (!ret) keyChild.ClearKeyData(); return ret; } EllSwiftPubKey CKey::EllSwiftCreate(Span<const std::byte> ent32) const { - assert(fValid); + assert(keydata); assert(ent32.size() == 32); std::array<std::byte, EllSwiftPubKey::size()> encoded_pubkey; auto success = secp256k1_ellswift_create(secp256k1_context_sign, UCharCast(encoded_pubkey.data()), - keydata.data(), + keydata->data(), UCharCast(ent32.data())); // Should always succeed for valid keys (asserted above). @@ -350,7 +351,7 @@ EllSwiftPubKey CKey::EllSwiftCreate(Span<const std::byte> ent32) const ECDHSecret CKey::ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift, const EllSwiftPubKey& our_ellswift, bool initiating) const { - assert(fValid); + assert(keydata); ECDHSecret output; // BIP324 uses the initiator as party A, and the responder as party B. Remap the inputs @@ -359,7 +360,7 @@ ECDHSecret CKey::ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift, c UCharCast(output.data()), UCharCast(initiating ? our_ellswift.data() : their_ellswift.data()), UCharCast(initiating ? their_ellswift.data() : our_ellswift.data()), - keydata.data(), + keydata->data(), initiating ? 0 : 1, secp256k1_ellswift_xdh_hash_function_bip324, nullptr); @@ -46,57 +46,77 @@ public: "COMPRESSED_SIZE is larger than SIZE"); private: - //! Whether this private key is valid. We check for correctness when modifying the key - //! data, so fValid should always correspond to the actual state. - bool fValid{false}; + /** Internal data container for private key material. */ + using KeyType = std::array<unsigned char, 32>; //! Whether the public key corresponding to this private key is (to be) compressed. bool fCompressed{false}; - //! The actual byte data - std::vector<unsigned char, secure_allocator<unsigned char> > keydata; + //! The actual byte data. nullptr for invalid keys. + secure_unique_ptr<KeyType> keydata; //! Check whether the 32-byte array pointed to by vch is valid keydata. bool static Check(const unsigned char* vch); + void MakeKeyData() + { + if (!keydata) keydata = make_secure_unique<KeyType>(); + } + + void ClearKeyData() + { + keydata.reset(); + } + public: - //! Construct an invalid private key. - CKey() + CKey() noexcept = default; + CKey(CKey&&) noexcept = default; + CKey& operator=(CKey&&) noexcept = default; + + CKey& operator=(const CKey& other) { - // Important: vch must be 32 bytes in length to not break serialization - keydata.resize(32); + if (other.keydata) { + MakeKeyData(); + *keydata = *other.keydata; + } else { + ClearKeyData(); + } + fCompressed = other.fCompressed; + return *this; } + CKey(const CKey& other) { *this = other; } + friend bool operator==(const CKey& a, const CKey& b) { return a.fCompressed == b.fCompressed && a.size() == b.size() && - memcmp(a.keydata.data(), b.keydata.data(), a.size()) == 0; + memcmp(a.data(), b.data(), a.size()) == 0; } //! Initialize using begin and end iterators to byte data. template <typename T> void Set(const T pbegin, const T pend, bool fCompressedIn) { - if (size_t(pend - pbegin) != keydata.size()) { - fValid = false; + if (size_t(pend - pbegin) != std::tuple_size_v<KeyType>) { + ClearKeyData(); } else if (Check(&pbegin[0])) { - memcpy(keydata.data(), (unsigned char*)&pbegin[0], keydata.size()); - fValid = true; + MakeKeyData(); + memcpy(keydata->data(), (unsigned char*)&pbegin[0], keydata->size()); fCompressed = fCompressedIn; } else { - fValid = false; + ClearKeyData(); } } //! Simple read-only vector-like interface. - unsigned int size() const { return (fValid ? keydata.size() : 0); } - const std::byte* data() const { return reinterpret_cast<const std::byte*>(keydata.data()); } - const unsigned char* begin() const { return keydata.data(); } - const unsigned char* end() const { return keydata.data() + size(); } + unsigned int size() const { return keydata ? keydata->size() : 0; } + const std::byte* data() const { return keydata ? reinterpret_cast<const std::byte*>(keydata->data()) : nullptr; } + const unsigned char* begin() const { return keydata ? keydata->data() : nullptr; } + const unsigned char* end() const { return begin() + size(); } //! Check whether this private key is valid. - bool IsValid() const { return fValid; } + bool IsValid() const { return !!keydata; } //! Check whether the public key corresponding to this private key is (to be) compressed. bool IsCompressed() const { return fCompressed; } diff --git a/src/net.cpp b/src/net.cpp index 0c6dac572d..6b2ef5f43d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -439,7 +439,7 @@ static CAddress GetBindAddress(const Sock& sock) return addr_bind; } -CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type) +CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type, bool use_v2transport) { AssertLockNotHeld(m_unused_i2p_sessions_mutex); assert(conn_type != ConnectionType::INBOUND); @@ -457,7 +457,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } } - LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "trying connection %s lastseen=%.1fhrs\n", + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "trying %s connection %s lastseen=%.1fhrs\n", + use_v2transport ? "v2" : "v1", pszDest ? pszDest : addrConnect.ToStringAddrPort(), Ticks<HoursDouble>(pszDest ? 0h : Now<NodeSeconds>() - addrConnect.nTime)); @@ -580,6 +581,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo CNodeOptions{ .i2p_sam_session = std::move(i2p_transient_session), .recv_flood_size = nReceiveFloodSize, + .use_v2transport = use_v2transport, }); pnode->AddRef(); @@ -665,6 +667,9 @@ void CNode::CopyStats(CNodeStats& stats) LOCK(cs_vRecv); X(mapRecvBytesPerMsgType); X(nRecvBytes); + Transport::Info info = m_transport->GetInfo(); + stats.m_transport_type = info.transport_type; + if (info.session_id) stats.m_session_id = HexStr(*info.session_id); } X(m_permission_flags); @@ -732,6 +737,11 @@ V1Transport::V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noex Reset(); } +Transport::Info V1Transport::GetInfo() const noexcept +{ + return {.transport_type = TransportProtocolType::V1, .session_id = {}}; +} + int V1Transport::readHeader(Span<const uint8_t> msg_bytes) { AssertLockHeld(m_recv_mutex); @@ -855,7 +865,7 @@ bool V1Transport::SetMessageToSend(CSerializedNetMsg& msg) noexcept // serialize header m_header_to_send.clear(); - CVectorWriter{SER_NETWORK, INIT_PROTO_VERSION, m_header_to_send, 0, hdr}; + CVectorWriter{INIT_PROTO_VERSION, m_header_to_send, 0, hdr}; // update state m_message_to_send = std::move(msg); @@ -1002,8 +1012,7 @@ void V2Transport::StartSendingHandshake() noexcept m_send_buffer.resize(EllSwiftPubKey::size() + m_send_garbage.size()); std::copy(std::begin(m_cipher.GetOurPubKey()), std::end(m_cipher.GetOurPubKey()), MakeWritableByteSpan(m_send_buffer).begin()); std::copy(m_send_garbage.begin(), m_send_garbage.end(), m_send_buffer.begin() + EllSwiftPubKey::size()); - // We cannot wipe m_send_garbage as it will still be used to construct the garbage - // authentication packet. + // We cannot wipe m_send_garbage as it will still be used as AAD later in the handshake. } V2Transport::V2Transport(NodeId nodeid, bool initiating, int type_in, int version_in, const CKey& key, Span<const std::byte> ent32, std::vector<uint8_t> garbage) noexcept : @@ -1037,9 +1046,6 @@ void V2Transport::SetReceiveState(RecvState recv_state) noexcept Assume(recv_state == RecvState::GARB_GARBTERM); break; case RecvState::GARB_GARBTERM: - Assume(recv_state == RecvState::GARBAUTH); - break; - case RecvState::GARBAUTH: Assume(recv_state == RecvState::VERSION); break; case RecvState::VERSION: @@ -1171,24 +1177,15 @@ bool V2Transport::ProcessReceivedKeyBytes() noexcept m_cipher.GetSendGarbageTerminator().end(), MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::GARBAGE_TERMINATOR_LEN).begin()); - // Construct garbage authentication packet in the send buffer (using the garbage data which - // is still there). - m_send_buffer.resize(m_send_buffer.size() + BIP324Cipher::EXPANSION); - m_cipher.Encrypt( - /*contents=*/{}, - /*aad=*/MakeByteSpan(m_send_garbage), - /*ignore=*/false, - /*output=*/MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::EXPANSION)); - // We no longer need the garbage. - ClearShrink(m_send_garbage); - - // Construct version packet in the send buffer. + // Construct version packet in the send buffer, with the sent garbage data as AAD. m_send_buffer.resize(m_send_buffer.size() + BIP324Cipher::EXPANSION + VERSION_CONTENTS.size()); m_cipher.Encrypt( /*contents=*/VERSION_CONTENTS, - /*aad=*/{}, + /*aad=*/MakeByteSpan(m_send_garbage), /*ignore=*/false, /*output=*/MakeWritableByteSpan(m_send_buffer).last(BIP324Cipher::EXPANSION + VERSION_CONTENTS.size())); + // We no longer need the garbage. + ClearShrink(m_send_garbage); } else { // We still have to receive more key bytes. } @@ -1202,11 +1199,11 @@ bool V2Transport::ProcessReceivedGarbageBytes() noexcept Assume(m_recv_buffer.size() <= MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN); if (m_recv_buffer.size() >= BIP324Cipher::GARBAGE_TERMINATOR_LEN) { if (MakeByteSpan(m_recv_buffer).last(BIP324Cipher::GARBAGE_TERMINATOR_LEN) == m_cipher.GetReceiveGarbageTerminator()) { - // Garbage terminator received. Switch to receiving garbage authentication packet. - m_recv_garbage = std::move(m_recv_buffer); - m_recv_garbage.resize(m_recv_garbage.size() - BIP324Cipher::GARBAGE_TERMINATOR_LEN); + // Garbage terminator received. Store garbage to authenticate it as AAD later. + m_recv_aad = std::move(m_recv_buffer); + m_recv_aad.resize(m_recv_aad.size() - BIP324Cipher::GARBAGE_TERMINATOR_LEN); m_recv_buffer.clear(); - SetReceiveState(RecvState::GARBAUTH); + SetReceiveState(RecvState::VERSION); } else if (m_recv_buffer.size() == MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN) { // We've reached the maximum length for garbage + garbage terminator, and the // terminator still does not match. Abort. @@ -1225,8 +1222,7 @@ bool V2Transport::ProcessReceivedGarbageBytes() noexcept bool V2Transport::ProcessReceivedPacketBytes() noexcept { AssertLockHeld(m_recv_mutex); - Assume(m_recv_state == RecvState::GARBAUTH || m_recv_state == RecvState::VERSION || - m_recv_state == RecvState::APP); + Assume(m_recv_state == RecvState::VERSION || m_recv_state == RecvState::APP); // The maximum permitted contents length for a packet, consisting of: // - 0x00 byte: indicating long message type encoding @@ -1249,45 +1245,37 @@ bool V2Transport::ProcessReceivedPacketBytes() noexcept // as GetMaxBytesToProcess only allows up to LENGTH_LEN into the buffer before that point. m_recv_decode_buffer.resize(m_recv_len); bool ignore{false}; - Span<const std::byte> aad; - if (m_recv_state == RecvState::GARBAUTH) aad = MakeByteSpan(m_recv_garbage); bool ret = m_cipher.Decrypt( /*input=*/MakeByteSpan(m_recv_buffer).subspan(BIP324Cipher::LENGTH_LEN), - /*aad=*/aad, + /*aad=*/MakeByteSpan(m_recv_aad), /*ignore=*/ignore, /*contents=*/MakeWritableByteSpan(m_recv_decode_buffer)); if (!ret) { LogPrint(BCLog::NET, "V2 transport error: packet decryption failure (%u bytes), peer=%d\n", m_recv_len, m_nodeid); return false; } + // We have decrypted a valid packet with the AAD we expected, so clear the expected AAD. + ClearShrink(m_recv_aad); // Feed the last 4 bytes of the Poly1305 authentication tag (and its timing) into our RNG. RandAddEvent(ReadLE32(m_recv_buffer.data() + m_recv_buffer.size() - 4)); - // At this point we have a valid packet decrypted into m_recv_decode_buffer. Depending on - // the current state, decide what to do with it. - switch (m_recv_state) { - case RecvState::GARBAUTH: - // Ignore flag does not matter for garbage authentication. Any valid packet functions - // as authentication. Receive and process the version packet next. - SetReceiveState(RecvState::VERSION); - ClearShrink(m_recv_garbage); - break; - case RecvState::VERSION: - if (!ignore) { + // At this point we have a valid packet decrypted into m_recv_decode_buffer. If it's not a + // decoy, which we simply ignore, use the current state to decide what to do with it. + if (!ignore) { + switch (m_recv_state) { + case RecvState::VERSION: // Version message received; transition to application phase. The contents is // ignored, but can be used for future extensions. SetReceiveState(RecvState::APP); - } - break; - case RecvState::APP: - if (!ignore) { + break; + case RecvState::APP: // Application message decrypted correctly. It can be extracted using GetMessage(). SetReceiveState(RecvState::APP_READY); + break; + default: + // Any other state is invalid (this function should not have been called). + Assume(false); } - break; - default: - // Any other state is invalid (this function should not have been called). - Assume(false); } // Wipe the receive buffer where the next packet will be received into. ClearShrink(m_recv_buffer); @@ -1323,7 +1311,6 @@ size_t V2Transport::GetMaxBytesToProcess() noexcept case RecvState::GARB_GARBTERM: // Process garbage bytes one by one (because terminator may appear anywhere). return 1; - case RecvState::GARBAUTH: case RecvState::VERSION: case RecvState::APP: // These three states all involve decoding a packet. Process the length descriptor first, @@ -1377,7 +1364,6 @@ bool V2Transport::ReceivedBytes(Span<const uint8_t>& msg_bytes) noexcept // bytes). m_recv_buffer.reserve(MAX_GARBAGE_LEN + BIP324Cipher::GARBAGE_TERMINATOR_LEN); break; - case RecvState::GARBAUTH: case RecvState::VERSION: case RecvState::APP: { // During states where a packet is being received, as much as is expected but never @@ -1421,7 +1407,6 @@ bool V2Transport::ReceivedBytes(Span<const uint8_t>& msg_bytes) noexcept if (!ProcessReceivedGarbageBytes()) return false; break; - case RecvState::GARBAUTH: case RecvState::VERSION: case RecvState::APP: if (!ProcessReceivedPacketBytes()) return false; @@ -1567,8 +1552,15 @@ void V2Transport::MarkBytesSent(size_t bytes_sent) noexcept LOCK(m_send_mutex); if (m_send_state == SendState::V1) return m_v1_fallback.MarkBytesSent(bytes_sent); + if (m_send_state == SendState::AWAITING_KEY && m_send_pos == 0 && bytes_sent > 0) { + LogPrint(BCLog::NET, "start sending v2 handshake to peer=%d\n", m_nodeid); + } + m_send_pos += bytes_sent; Assume(m_send_pos <= m_send_buffer.size()); + if (m_send_pos >= CMessageHeader::HEADER_SIZE) { + m_sent_v1_header_worth = true; + } // Wipe the buffer when everything is sent. if (m_send_pos == m_send_buffer.size()) { m_send_pos = 0; @@ -1576,6 +1568,23 @@ void V2Transport::MarkBytesSent(size_t bytes_sent) noexcept } } +bool V2Transport::ShouldReconnectV1() const noexcept +{ + AssertLockNotHeld(m_send_mutex); + AssertLockNotHeld(m_recv_mutex); + // Only outgoing connections need reconnection. + if (!m_initiating) return false; + + LOCK(m_recv_mutex); + // We only reconnect in the very first state and when the receive buffer is empty. Together + // these conditions imply nothing has been received so far. + if (m_recv_state != RecvState::KEY) return false; + if (!m_recv_buffer.empty()) return false; + // Check if we've sent enough for the other side to disconnect us (if it was V1). + LOCK(m_send_mutex); + return m_sent_v1_header_worth; +} + size_t V2Transport::GetSendMemoryUsage() const noexcept { AssertLockNotHeld(m_send_mutex); @@ -1585,6 +1594,27 @@ size_t V2Transport::GetSendMemoryUsage() const noexcept return sizeof(m_send_buffer) + memusage::DynamicUsage(m_send_buffer); } +Transport::Info V2Transport::GetInfo() const noexcept +{ + AssertLockNotHeld(m_recv_mutex); + LOCK(m_recv_mutex); + if (m_recv_state == RecvState::V1) return m_v1_fallback.GetInfo(); + + Transport::Info info; + + // Do not report v2 and session ID until the version packet has been received + // and verified (confirming that the other side very likely has the same keys as us). + if (m_recv_state != RecvState::KEY_MAYBE_V1 && m_recv_state != RecvState::KEY && + m_recv_state != RecvState::GARB_GARBTERM && m_recv_state != RecvState::VERSION) { + info.transport_type = TransportProtocolType::V2; + info.session_id = uint256(MakeUCharSpan(m_cipher.GetSessionID())); + } else { + info.transport_type = TransportProtocolType::DETECTING; + } + + return info; +} + std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const { auto it = node.vSendMsg.begin(); @@ -1634,7 +1664,9 @@ std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const // Notify transport that bytes have been processed. node.m_transport->MarkBytesSent(nBytes); // Update statistics per message type. - node.AccountForSentBytes(msg_type, nBytes); + if (!msg_type.empty()) { // don't report v2 handshake bytes for now + node.AccountForSentBytes(msg_type, nBytes); + } nSentSize += nBytes; if ((size_t)nBytes != data.size()) { // could not send full message; stop sending more @@ -1817,6 +1849,10 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, } const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); + // The V2Transport transparently falls back to V1 behavior when an incoming V1 connection is + // detected, so use it whenever we signal NODE_P2P_V2. + const bool use_v2transport(nodeServices & NODE_P2P_V2); + CNode* pnode = new CNode(id, std::move(sock), addr, @@ -1830,6 +1866,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, .permission_flags = permission_flags, .prefer_evict = discouraged, .recv_flood_size = nReceiveFloodSize, + .use_v2transport = use_v2transport, }); pnode->AddRef(); m_msgproc->InitializeNode(*pnode, nodeServices); @@ -1878,12 +1915,19 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ CSemaphoreGrant grant(*semOutbound, true); if (!grant) return false; - OpenNetworkConnection(CAddress(), false, &grant, address.c_str(), conn_type); + OpenNetworkConnection(CAddress(), false, std::move(grant), address.c_str(), conn_type, /*use_v2transport=*/false); return true; } void CConnman::DisconnectNodes() { + AssertLockNotHeld(m_nodes_mutex); + AssertLockNotHeld(m_reconnections_mutex); + + // Use a temporary variable to accumulate desired reconnections, so we don't need + // m_reconnections_mutex while holding m_nodes_mutex. + decltype(m_reconnections) reconnections_to_add; + { LOCK(m_nodes_mutex); @@ -1906,6 +1950,19 @@ void CConnman::DisconnectNodes() // remove from m_nodes m_nodes.erase(remove(m_nodes.begin(), m_nodes.end(), pnode), m_nodes.end()); + // Add to reconnection list if appropriate. We don't reconnect right here, because + // the creation of a connection is a blocking operation (up to several seconds), + // and we don't want to hold up the socket handler thread for that long. + if (pnode->m_transport->ShouldReconnectV1()) { + reconnections_to_add.push_back({ + .addr_connect = pnode->addr, + .grant = std::move(pnode->grantOutbound), + .destination = pnode->m_dest, + .conn_type = pnode->m_conn_type, + .use_v2transport = false}); + LogPrint(BCLog::NET, "retrying with v1 transport protocol for peer=%d\n", pnode->GetId()); + } + // release outbound grant (if any) pnode->grantOutbound.Release(); @@ -1933,6 +1990,11 @@ void CConnman::DisconnectNodes() } } } + { + // Move entries from reconnections_to_add to m_reconnections. + LOCK(m_reconnections_mutex); + m_reconnections.splice(m_reconnections.end(), std::move(reconnections_to_add)); + } } void CConnman::NotifyNumConnectionsChanged() @@ -2310,9 +2372,9 @@ void CConnman::ProcessAddrFetch() m_addr_fetches.pop_front(); } CAddress addr; - CSemaphoreGrant grant(*semOutbound, true); + CSemaphoreGrant grant(*semOutbound, /*fTry=*/true); if (grant) { - OpenNetworkConnection(addr, false, &grant, strDest.c_str(), ConnectionType::ADDR_FETCH); + OpenNetworkConnection(addr, false, std::move(grant), strDest.c_str(), ConnectionType::ADDR_FETCH, /*use_v2transport=*/false); } } @@ -2405,6 +2467,7 @@ bool CConnman::MaybePickPreferredNetwork(std::optional<Network>& network) void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { AssertLockNotHeld(m_unused_i2p_sessions_mutex); + AssertLockNotHeld(m_reconnections_mutex); FastRandomContext rng; // Connect to specific addresses if (!connect.empty()) @@ -2414,7 +2477,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) for (const std::string& strAddr : connect) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), ConnectionType::MANUAL); + OpenNetworkConnection(addr, false, {}, strAddr.c_str(), ConnectionType::MANUAL, /*use_v2transport=*/false); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -2448,6 +2511,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + PerformReconnections(); + CSemaphoreGrant grant(*semOutbound); if (interruptNet) return; @@ -2468,7 +2533,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Perform cheap checks before locking a mutex. else if (!dnsseed && !use_seednodes) { LOCK(m_added_nodes_mutex); - if (m_added_nodes.empty()) { + if (m_added_node_params.empty()) { add_fixed_seeds_now = true; LogPrintf("Adding fixed seeds as -dnsseed=0 (or IPv4/IPv6 connections are disabled via -onlynet) and neither -addnode nor -seednode are provided\n"); } @@ -2717,7 +2782,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Don't record addrman failure attempts when node is offline. This can be identified since all local // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; - OpenNetworkConnection(addrConnect, count_failures, &grant, /*strDest=*/nullptr, conn_type); + // Use BIP324 transport when both us and them have NODE_V2_P2P set. + const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); + OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*strDest=*/nullptr, conn_type, use_v2transport); } } } @@ -2739,11 +2806,11 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const { std::vector<AddedNodeInfo> ret; - std::list<std::string> lAddresses(0); + std::list<AddedNodeParams> lAddresses(0); { LOCK(m_added_nodes_mutex); - ret.reserve(m_added_nodes.size()); - std::copy(m_added_nodes.cbegin(), m_added_nodes.cend(), std::back_inserter(lAddresses)); + ret.reserve(m_added_node_params.size()); + std::copy(m_added_node_params.cbegin(), m_added_node_params.cend(), std::back_inserter(lAddresses)); } @@ -2763,9 +2830,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const } } - for (const std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode, GetDefaultPort(strAddNode))); - AddedNodeInfo addedNode{strAddNode, CService(), false, false}; + for (const auto& addr : lAddresses) { + CService service(LookupNumeric(addr.m_added_node, GetDefaultPort(addr.m_added_node))); + AddedNodeInfo addedNode{addr, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port auto it = mapConnected.find(service); @@ -2776,7 +2843,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const } } else { // strAddNode is a name - auto it = mapConnectedByName.find(strAddNode); + auto it = mapConnectedByName.find(addr.m_added_node); if (it != mapConnectedByName.end()) { addedNode.resolvedAddress = it->second.second; addedNode.fConnected = true; @@ -2792,6 +2859,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const void CConnman::ThreadOpenAddedConnections() { AssertLockNotHeld(m_unused_i2p_sessions_mutex); + AssertLockNotHeld(m_reconnections_mutex); while (true) { CSemaphoreGrant grant(*semAddnode); @@ -2799,26 +2867,28 @@ void CConnman::ThreadOpenAddedConnections() bool tried = false; for (const AddedNodeInfo& info : vInfo) { if (!info.fConnected) { - if (!grant.TryAcquire()) { + if (!grant) { // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting // the addednodeinfo state might change. break; } tried = true; CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, &grant, info.strAddedNode.c_str(), ConnectionType::MANUAL); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) - return; + OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); } } // Retry every 60 seconds if a connection was attempted, otherwise two seconds if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) return; + // See if any reconnections are desired. + PerformReconnections(); } } // if successful, this moves the passed grant to the constructed node -void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, ConnectionType conn_type) +void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant&& grant_outbound, const char *pszDest, ConnectionType conn_type, bool use_v2transport) { AssertLockNotHeld(m_unused_i2p_sessions_mutex); assert(conn_type != ConnectionType::INBOUND); @@ -2840,12 +2910,11 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai } else if (FindNode(std::string(pszDest))) return; - CNode* pnode = ConnectNode(addrConnect, pszDest, fCountFailure, conn_type); + CNode* pnode = ConnectNode(addrConnect, pszDest, fCountFailure, conn_type, use_v2transport); if (!pnode) return; - if (grantOutbound) - grantOutbound->MoveTo(pnode->grantOutbound); + pnode->grantOutbound = std::move(grant_outbound); m_msgproc->InitializeNode(*pnode, nLocalServices); { @@ -3398,23 +3467,23 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres return cache_entry.m_addrs_response_cache; } -bool CConnman::AddNode(const std::string& strNode) +bool CConnman::AddNode(const AddedNodeParams& add) { LOCK(m_added_nodes_mutex); - for (const std::string& it : m_added_nodes) { - if (strNode == it) return false; + for (const auto& it : m_added_node_params) { + if (add.m_added_node == it.m_added_node) return false; } - m_added_nodes.push_back(strNode); + m_added_node_params.push_back(add); return true; } bool CConnman::RemoveAddedNode(const std::string& strNode) { LOCK(m_added_nodes_mutex); - for(std::vector<std::string>::iterator it = m_added_nodes.begin(); it != m_added_nodes.end(); ++it) { - if (strNode == *it) { - m_added_nodes.erase(it); + for (auto it = m_added_node_params.begin(); it != m_added_node_params.end(); ++it) { + if (strNode == it->m_added_node) { + m_added_node_params.erase(it); return true; } } @@ -3602,6 +3671,15 @@ ServiceFlags CConnman::GetLocalServices() const return nLocalServices; } +static std::unique_ptr<Transport> MakeTransport(NodeId id, bool use_v2transport, bool inbound) noexcept +{ + if (use_v2transport) { + return std::make_unique<V2Transport>(id, /*initiating=*/!inbound, SER_NETWORK, INIT_PROTO_VERSION); + } else { + return std::make_unique<V1Transport>(id, SER_NETWORK, INIT_PROTO_VERSION); + } +} + CNode::CNode(NodeId idIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, @@ -3612,13 +3690,14 @@ CNode::CNode(NodeId idIn, ConnectionType conn_type_in, bool inbound_onion, CNodeOptions&& node_opts) - : m_transport{std::make_unique<V1Transport>(idIn, SER_NETWORK, INIT_PROTO_VERSION)}, + : m_transport{MakeTransport(idIn, node_opts.use_v2transport, conn_type_in == ConnectionType::INBOUND)}, m_permission_flags{node_opts.permission_flags}, m_sock{sock}, m_connected{GetTime<std::chrono::seconds>()}, addr{addrIn}, addrBind{addrBindIn}, m_addr_name{addrNameIn.empty() ? addr.ToStringAddrPort() : addrNameIn}, + m_dest(addrNameIn), m_inbound_onion{inbound_onion}, m_prefer_evict{node_opts.prefer_evict}, nKeyedNetGroup{nKeyedNetGroupIn}, @@ -3749,6 +3828,33 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& address) const return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup).Finalize(); } +void CConnman::PerformReconnections() +{ + AssertLockNotHeld(m_reconnections_mutex); + AssertLockNotHeld(m_unused_i2p_sessions_mutex); + while (true) { + // Move first element of m_reconnections to todo (avoiding an allocation inside the lock). + decltype(m_reconnections) todo; + { + LOCK(m_reconnections_mutex); + if (m_reconnections.empty()) break; + todo.splice(todo.end(), m_reconnections, m_reconnections.begin()); + } + + auto& item = *todo.begin(); + OpenNetworkConnection(item.addr_connect, + // We only reconnect if the first attempt to connect succeeded at + // connection time, but then failed after the CNode object was + // created. Since we already know connecting is possible, do not + // count failure to reconnect. + /*fCountFailure=*/false, + std::move(item.grant), + item.destination.empty() ? nullptr : item.destination.c_str(), + item.conn_type, + item.use_v2transport); + } +} + // Dump binary message to file, with timestamp. static void CaptureMessageToFile(const CAddress& addr, const std::string& msg_type, @@ -94,11 +94,17 @@ static constexpr bool DEFAULT_FIXEDSEEDS{true}; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; +static constexpr bool DEFAULT_V2_TRANSPORT{false}; + typedef int64_t NodeId; -struct AddedNodeInfo -{ - std::string strAddedNode; +struct AddedNodeParams { + std::string m_added_node; + bool m_use_v2transport; +}; + +struct AddedNodeInfo { + AddedNodeParams m_params; CService resolvedAddress; bool fConnected; bool fInbound; @@ -226,6 +232,10 @@ public: Network m_network; uint32_t m_mapped_as; ConnectionType m_conn_type; + /** Transport protocol type. */ + TransportProtocolType m_transport_type; + /** BIP324 session id string in hex, if any. */ + std::string m_session_id; }; @@ -262,6 +272,15 @@ class Transport { public: virtual ~Transport() {} + struct Info + { + TransportProtocolType transport_type; + std::optional<uint256> session_id; + }; + + /** Retrieve information about this transport. */ + virtual Info GetInfo() const noexcept = 0; + // 1. Receiver side functions, for decoding bytes received on the wire into transport protocol // agnostic CNetMessage (message type & payload) objects. @@ -355,6 +374,11 @@ public: /** Return the memory usage of this transport attributable to buffered data to send. */ virtual size_t GetSendMemoryUsage() const noexcept = 0; + + // 3. Miscellaneous functions. + + /** Whether upon disconnections, a reconnect with V1 is warranted. */ + virtual bool ShouldReconnectV1() const noexcept = 0; }; class V1Transport final : public Transport @@ -415,6 +439,8 @@ public: return WITH_LOCK(m_recv_mutex, return CompleteInternal()); } + Info GetInfo() const noexcept override; + bool ReceivedBytes(Span<const uint8_t>& msg_bytes) override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex) { AssertLockNotHeld(m_recv_mutex); @@ -434,6 +460,7 @@ public: BytesToSend GetBytesToSend(bool have_next_message) const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); void MarkBytesSent(size_t bytes_sent) noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); size_t GetSendMemoryUsage() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); + bool ShouldReconnectV1() const noexcept override { return false; } }; class V2Transport final : public Transport @@ -460,10 +487,10 @@ private: * * start(responder) * | - * | start(initiator) /---------\ - * | | | | - * v v v | - * KEY_MAYBE_V1 -> KEY -> GARB_GARBTERM -> GARBAUTH -> VERSION -> APP -> APP_READY + * | start(initiator) /---------\ + * | | | | + * v v v | + * KEY_MAYBE_V1 -> KEY -> GARB_GARBTERM -> VERSION -> APP -> APP_READY * | * \-------> V1 */ @@ -485,24 +512,19 @@ private: /** Garbage and garbage terminator. * * Whenever a byte is received, the last 16 bytes are compared with the expected garbage - * terminator. When that happens, the state becomes GARBAUTH. If no matching terminator is + * terminator. When that happens, the state becomes VERSION. If no matching terminator is * received in 4111 bytes (4095 for the maximum garbage length, and 16 bytes for the * terminator), the connection aborts. */ GARB_GARBTERM, - /** Garbage authentication packet. - * - * A packet is received, and decrypted/verified with AAD set to the garbage received during - * the GARB_GARBTERM state. If that succeeds, the state becomes VERSION. If it fails the - * connection aborts. */ - GARBAUTH, - /** Version packet. * - * A packet is received, and decrypted/verified. If that succeeds, the state becomes APP, - * and the decrypted contents is interpreted as version negotiation (currently, that means - * ignoring it, but it can be used for negotiating future extensions). If it fails, the - * connection aborts. */ + * A packet is received, and decrypted/verified. If that fails, the connection aborts. The + * first received packet in this state (whether it's a decoy or not) is expected to + * authenticate the garbage received during the GARB_GARBTERM state as associated + * authenticated data (AAD). The first non-decoy packet in this state is interpreted as + * version negotiation (currently, that means ignoring the contents, but it can be used for + * negotiating future extensions), and afterwards the state becomes APP. */ VERSION, /** Application packet. @@ -556,9 +578,9 @@ private: /** Normal sending state. * * In this state, the ciphers are initialized, so packets can be sent. When this state is - * entered, the garbage terminator, garbage authentication packet, and version - * packet are appended to the send buffer (in addition to the key and garbage which may - * still be there). In this state a message can be provided if the send buffer is empty. */ + * entered, the garbage terminator and version packet are appended to the send buffer (in + * addition to the key and garbage which may still be there). In this state a message can be + * provided if the send buffer is empty. */ READY, /** This transport is using v1 fallback. @@ -578,13 +600,13 @@ private: /** Lock for receiver-side fields. */ mutable Mutex m_recv_mutex ACQUIRED_BEFORE(m_send_mutex); - /** In {GARBAUTH, VERSION, APP}, the decrypted packet length, if m_recv_buffer.size() >= + /** In {VERSION, APP}, the decrypted packet length, if m_recv_buffer.size() >= * BIP324Cipher::LENGTH_LEN. Unspecified otherwise. */ uint32_t m_recv_len GUARDED_BY(m_recv_mutex) {0}; /** Receive buffer; meaning is determined by m_recv_state. */ std::vector<uint8_t> m_recv_buffer GUARDED_BY(m_recv_mutex); - /** During GARBAUTH, the garbage received during GARB_GARBTERM. */ - std::vector<uint8_t> m_recv_garbage GUARDED_BY(m_recv_mutex); + /** AAD expected in next received packet (currently used only for garbage). */ + std::vector<uint8_t> m_recv_aad GUARDED_BY(m_recv_mutex); /** Buffer to put decrypted contents in, for converting to CNetMessage. */ std::vector<uint8_t> m_recv_decode_buffer GUARDED_BY(m_recv_mutex); /** Deserialization type. */ @@ -607,6 +629,8 @@ private: std::string m_send_type GUARDED_BY(m_send_mutex); /** Current sender state. */ SendState m_send_state GUARDED_BY(m_send_mutex); + /** Whether we've sent at least 24 bytes (which would trigger disconnect for V1 peers). */ + bool m_sent_v1_header_worth GUARDED_BY(m_send_mutex) {false}; /** Change the receive state. */ void SetReceiveState(RecvState recv_state) noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); @@ -624,7 +648,7 @@ private: bool ProcessReceivedKeyBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex, !m_send_mutex); /** Process bytes in m_recv_buffer, while in GARB_GARBTERM state. */ bool ProcessReceivedGarbageBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); - /** Process bytes in m_recv_buffer, while in GARBAUTH/VERSION/APP state. */ + /** Process bytes in m_recv_buffer, while in VERSION/APP state. */ bool ProcessReceivedPacketBytes() noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); public: @@ -652,6 +676,10 @@ public: BytesToSend GetBytesToSend(bool have_next_message) const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); void MarkBytesSent(size_t bytes_sent) noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); size_t GetSendMemoryUsage() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); + + // Miscellaneous functions. + bool ShouldReconnectV1() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex, !m_send_mutex); + Info GetInfo() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); }; struct CNodeOptions @@ -660,6 +688,7 @@ struct CNodeOptions std::unique_ptr<i2p::sam::Session> i2p_sam_session = nullptr; bool prefer_evict = false; size_t recv_flood_size{DEFAULT_MAXRECEIVEBUFFER * 1000}; + bool use_v2transport = false; }; /** Information about a peer */ @@ -704,6 +733,8 @@ public: // Bind address of our side of the connection const CAddress addrBind; const std::string m_addr_name; + /** The pszDest argument provided to ConnectNode(). Only used for reconnections. */ + const std::string m_dest; //! Whether this peer is an inbound onion, i.e. connected via our Tor onion service. const bool m_inbound_onion; std::atomic<int> nVersion{0}; @@ -1077,7 +1108,11 @@ public: vWhitelistedRange = connOptions.vWhitelistedRange; { LOCK(m_added_nodes_mutex); - m_added_nodes = connOptions.m_added_nodes; + + for (const std::string& added_node : connOptions.m_added_nodes) { + // -addnode cli arg does not currently have a way to signal BIP324 support + m_added_node_params.push_back({added_node, false}); + } } m_onion_binds = connOptions.onion_binds; } @@ -1101,7 +1136,7 @@ public: bool GetNetworkActive() const { return fNetworkActive; }; bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; void SetNetworkActive(bool active); - void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant* grantOutbound, const char* strDest, ConnectionType conn_type) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant&& grant_outbound, const char* strDest, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); bool CheckIncomingNonce(uint64_t nonce); // alias for thread safety annotations only, not defined @@ -1164,7 +1199,7 @@ public: // Count the number of block-relay-only peers we have over our limit. int GetExtraBlockRelayCount() const; - bool AddNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); std::vector<AddedNodeInfo> GetAddedNodeInfo() const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); @@ -1247,10 +1282,10 @@ private: bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const Options& options); - void ThreadOpenAddedConnections() EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_unused_i2p_sessions_mutex); + void ThreadOpenAddedConnections() EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex); void AddAddrFetch(const std::string& strDest) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex); void ProcessAddrFetch() EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_unused_i2p_sessions_mutex); - void ThreadOpenConnections(std::vector<std::string> connect) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_added_nodes_mutex, !m_nodes_mutex, !m_unused_i2p_sessions_mutex); + void ThreadOpenConnections(std::vector<std::string> connect) EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_added_nodes_mutex, !m_nodes_mutex, !m_unused_i2p_sessions_mutex, !m_reconnections_mutex); void ThreadMessageHandler() EXCLUSIVE_LOCKS_REQUIRED(!mutexMsgProc); void ThreadI2PAcceptIncoming(); void AcceptConnection(const ListenSocket& hListenSocket); @@ -1268,7 +1303,7 @@ private: const CAddress& addr_bind, const CAddress& addr); - void DisconnectNodes(); + void DisconnectNodes() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_nodes_mutex); void NotifyNumConnectionsChanged(); /** Return true if the peer is inactive and should be disconnected. */ bool InactivityCheck(const CNode& node) const; @@ -1300,7 +1335,7 @@ private: */ void SocketHandlerListening(const Sock::EventsPerSock& events_per_sock); - void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc); + void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex, !mutexMsgProc, !m_nodes_mutex, !m_reconnections_mutex); void ThreadDNSAddressSeed() EXCLUSIVE_LOCKS_REQUIRED(!m_addr_fetches_mutex, !m_nodes_mutex); uint64_t CalculateKeyedNetGroup(const CAddress& ad) const; @@ -1317,7 +1352,7 @@ private: bool AlreadyConnectedToAddress(const CAddress& addr); bool AttemptToEvictConnection(); - CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); + CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const; void DeleteNode(CNode* pnode); @@ -1389,7 +1424,10 @@ private: const NetGroupManager& m_netgroupman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); Mutex m_addr_fetches_mutex; - std::vector<std::string> m_added_nodes GUARDED_BY(m_added_nodes_mutex); + + // connection string and whether to use v2 p2p + std::vector<AddedNodeParams> m_added_node_params GUARDED_BY(m_added_nodes_mutex); + mutable Mutex m_added_nodes_mutex; std::vector<CNode*> m_nodes GUARDED_BY(m_nodes_mutex); std::list<CNode*> m_nodes_disconnected; @@ -1529,6 +1567,29 @@ private: std::queue<std::unique_ptr<i2p::sam::Session>> m_unused_i2p_sessions GUARDED_BY(m_unused_i2p_sessions_mutex); /** + * Mutex protecting m_reconnections. + */ + Mutex m_reconnections_mutex; + + /** Struct for entries in m_reconnections. */ + struct ReconnectionInfo + { + CAddress addr_connect; + CSemaphoreGrant grant; + std::string destination; + ConnectionType conn_type; + bool use_v2transport; + }; + + /** + * List of reconnections we have to make. + */ + std::list<ReconnectionInfo> m_reconnections GUARDED_BY(m_reconnections_mutex); + + /** Attempt reconnections, if m_reconnections non-empty. */ + void PerformReconnections() EXCLUSIVE_LOCKS_REQUIRED(!m_reconnections_mutex, !m_unused_i2p_sessions_mutex); + + /** * Cap on the size of `m_unused_i2p_sessions`, to ensure it does not * unexpectedly use too much memory. */ diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b046b3ac16..06086d6804 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -18,6 +18,7 @@ #include <index/blockfilterindex.h> #include <kernel/mempool_entry.h> #include <logging.h> +#include <kernel/chain.h> #include <merkleblock.h> #include <netbase.h> #include <netmessagemaker.h> @@ -483,7 +484,7 @@ public: CTxMemPool& pool, Options opts); /** Overridden from CValidationInterface. */ - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override + void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override EXCLUSIVE_LOCKS_REQUIRED(!m_recent_confirmed_transactions_mutex); void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override EXCLUSIVE_LOCKS_REQUIRED(!m_recent_confirmed_transactions_mutex); @@ -892,6 +893,38 @@ private: */ void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Request blocks for the background chainstate, if one is in use. */ + void TryDownloadingHistoricalBlocks(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, const CBlockIndex* from_tip, const CBlockIndex* target_block) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * \brief Find next blocks to download from a peer after a starting block. + * + * \param vBlocks Vector of blocks to download which will be appended to. + * \param peer Peer which blocks will be downloaded from. + * \param state Pointer to the state of the peer. + * \param pindexWalk Pointer to the starting block to add to vBlocks. + * \param count Maximum number of blocks to allow in vBlocks. No more + * blocks will be added if it reaches this size. + * \param nWindowEnd Maximum height of blocks to allow in vBlocks. No + * blocks will be added above this height. + * \param activeChain Optional pointer to a chain to compare against. If + * provided, any next blocks which are already contained + * in this chain will not be appended to vBlocks, but + * instead will be used to update the + * state->pindexLastCommonBlock pointer. + * \param nodeStaller Optional pointer to a NodeId variable that will receive + * the ID of another peer that might be causing this peer + * to stall. This is set to the ID of the peer which + * first requested the first in-flight block in the + * download window. It is only set if vBlocks is empty at + * the end of this function call and if increasing + * nWindowEnd by 1 would cause it to be non-empty (which + * indicates the download might be stalled because every + * block in the window is in flight and no other peer is + * trying to download the next block). + */ + void FindNextBlocks(std::vector<const CBlockIndex*>& vBlocks, const Peer& peer, CNodeState *state, const CBlockIndex *pindexWalk, unsigned int count, int nWindowEnd, const CChain* activeChain=nullptr, NodeId* nodeStaller=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /* Multimap used to preserve insertion order */ typedef std::multimap<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator>> BlockDownloadMap; BlockDownloadMap mapBlocksInFlight GUARDED_BY(cs_main); @@ -1312,6 +1345,7 @@ void PeerManagerImpl::UpdateBlockAvailability(NodeId nodeid, const uint256 &hash } } +// Logic for calculating which blocks to download from a given peer, given our current tip. void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) { if (count == 0) @@ -1341,12 +1375,47 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) return; - std::vector<const CBlockIndex*> vToFetch; const CBlockIndex *pindexWalk = state->pindexLastCommonBlock; // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last // linked block we have in common with this peer. The +1 is so we can detect stalling, namely if we would be able to // download that next block if the window were 1 larger. int nWindowEnd = state->pindexLastCommonBlock->nHeight + BLOCK_DOWNLOAD_WINDOW; + + FindNextBlocks(vBlocks, peer, state, pindexWalk, count, nWindowEnd, &m_chainman.ActiveChain(), &nodeStaller); +} + +void PeerManagerImpl::TryDownloadingHistoricalBlocks(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, const CBlockIndex *from_tip, const CBlockIndex* target_block) +{ + Assert(from_tip); + Assert(target_block); + + if (vBlocks.size() >= count) { + return; + } + + vBlocks.reserve(count); + CNodeState *state = Assert(State(peer.m_id)); + + if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->GetAncestor(target_block->nHeight) != target_block) { + // This peer can't provide us the complete series of blocks leading up to the + // assumeutxo snapshot base. + // + // Presumably this peer's chain has less work than our ActiveChain()'s tip, or else we + // will eventually crash when we try to reorg to it. Let other logic + // deal with whether we disconnect this peer. + // + // TODO at some point in the future, we might choose to request what blocks + // this peer does have from the historical chain, despite it not having a + // complete history beneath the snapshot base. + return; + } + + FindNextBlocks(vBlocks, peer, state, from_tip, count, std::min<int>(from_tip->nHeight + BLOCK_DOWNLOAD_WINDOW, target_block->nHeight)); +} + +void PeerManagerImpl::FindNextBlocks(std::vector<const CBlockIndex*>& vBlocks, const Peer& peer, CNodeState *state, const CBlockIndex *pindexWalk, unsigned int count, int nWindowEnd, const CChain* activeChain, NodeId* nodeStaller) +{ + std::vector<const CBlockIndex*> vToFetch; int nMaxHeight = std::min<int>(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); NodeId waitingfor = -1; while (pindexWalk->nHeight < nMaxHeight) { @@ -1374,8 +1443,8 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co // We wouldn't download this block or its descendants from this peer. return; } - if (pindex->nStatus & BLOCK_HAVE_DATA || m_chainman.ActiveChain().Contains(pindex)) { - if (pindex->HaveTxsDownloaded()) + if (pindex->nStatus & BLOCK_HAVE_DATA || (activeChain && activeChain->Contains(pindex))) { + if (activeChain && pindex->HaveTxsDownloaded()) state->pindexLastCommonBlock = pindex; } else if (!IsBlockRequested(pindex->GetBlockHash())) { // The block is not already downloaded, and not yet in flight. @@ -1383,7 +1452,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co // We reached the end of the window. if (vBlocks.size() == 0 && waitingfor != peer.m_id) { // We aren't able to fetch anything, but we would be if the download window was one larger. - nodeStaller = waitingfor; + if (nodeStaller) *nodeStaller = waitingfor; } return; } @@ -1843,11 +1912,30 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) * announcements for them. Also save the time of the last tip update and * possibly reduce dynamic block stalling timeout. */ -void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +void PeerManagerImpl::BlockConnected( + ChainstateRole role, + const std::shared_ptr<const CBlock>& pblock, + const CBlockIndex* pindex) { - m_orphanage.EraseForBlock(*pblock); + // Update this for all chainstate roles so that we don't mistakenly see peers + // helping us do background IBD as having a stale tip. m_last_tip_update = GetTime<std::chrono::seconds>(); + // In case the dynamic timeout was doubled once or more, reduce it slowly back to its default value + auto stalling_timeout = m_block_stalling_timeout.load(); + Assume(stalling_timeout >= BLOCK_STALLING_TIMEOUT_DEFAULT); + if (stalling_timeout != BLOCK_STALLING_TIMEOUT_DEFAULT) { + const auto new_timeout = std::max(std::chrono::duration_cast<std::chrono::seconds>(stalling_timeout * 0.85), BLOCK_STALLING_TIMEOUT_DEFAULT); + if (m_block_stalling_timeout.compare_exchange_strong(stalling_timeout, new_timeout)) { + LogPrint(BCLog::NET, "Decreased stalling timeout to %d seconds\n", count_seconds(new_timeout)); + } + } + + if (role == ChainstateRole::BACKGROUND) { + return; + } + m_orphanage.EraseForBlock(*pblock); + { LOCK(m_recent_confirmed_transactions_mutex); for (const auto& ptx : pblock->vtx) { @@ -1864,16 +1952,6 @@ void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); } } - - // In case the dynamic timeout was doubled once or more, reduce it slowly back to its default value - auto stalling_timeout = m_block_stalling_timeout.load(); - Assume(stalling_timeout >= BLOCK_STALLING_TIMEOUT_DEFAULT); - if (stalling_timeout != BLOCK_STALLING_TIMEOUT_DEFAULT) { - const auto new_timeout = std::max(std::chrono::duration_cast<std::chrono::seconds>(stalling_timeout * 0.85), BLOCK_STALLING_TIMEOUT_DEFAULT); - if (m_block_stalling_timeout.compare_exchange_strong(stalling_timeout, new_timeout)) { - LogPrint(BCLog::NET, "Decreased stalling timeout to %d seconds\n", count_seconds(new_timeout)); - } - } } void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) @@ -3507,13 +3585,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (!pfrom.IsInboundConn()) { + // Log succesful connections unconditionally for outbound, but not for inbound as those + // can be triggered by an attacker at high rate. + if (!pfrom.IsInboundConn() || LogAcceptCategory(BCLog::NET, BCLog::Level::Debug)) { const auto mapped_as{m_connman.GetMappedAS(pfrom.addr)}; - LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s%s (%s)\n", + LogPrintf("New %s %s peer connected: version: %d, blocks=%d, peer=%d%s%s\n", + pfrom.ConnectionTypeAsString(), + TransportTypeAsString(pfrom.m_transport->GetInfo().transport_type), pfrom.nVersion.load(), peer->m_starting_height, pfrom.GetId(), (fLogIPs ? strprintf(", peeraddr=%s", pfrom.addr.ToStringAddrPort()) : ""), - (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : ""), - pfrom.ConnectionTypeAsString()); + (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : "")); } if (pfrom.GetCommonVersion() >= SHORT_IDS_BLOCKS_VERSION) { @@ -5847,7 +5928,20 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller); + auto get_inflight_budget = [&state]() { + return std::max(0, MAX_BLOCKS_IN_TRANSIT_PER_PEER - static_cast<int>(state.vBlocksInFlight.size())); + }; + + // If a snapshot chainstate is in use, we want to find its next blocks + // before the background chainstate to prioritize getting to network tip. + FindNextBlocksToDownload(*peer, get_inflight_budget(), vToDownload, staller); + if (m_chainman.BackgroundSyncInProgress() && !IsLimitedPeer(*peer)) { + TryDownloadingHistoricalBlocks( + *peer, + get_inflight_budget(), + vToDownload, m_chainman.GetBackgroundSyncTip(), + Assert(m_chainman.GetSnapshotBaseBlock())); + } for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index 89fb4758f9..a121183aab 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -19,7 +19,7 @@ public: { CSerializedNetMsg msg; msg.m_type = std::move(msg_type); - CVectorWriter{ SER_NETWORK, nFlags | nVersion, msg.data, 0, std::forward<Args>(args)... }; + CVectorWriter{nFlags | nVersion, msg.data, 0, std::forward<Args>(args)...}; return msg; } diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index ae63d12ef7..5e61ed3100 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -10,6 +10,7 @@ #include <dbwrapper.h> #include <flatfile.h> #include <hash.h> +#include <kernel/chain.h> #include <kernel/chainparams.h> #include <kernel/messagestartchars.h> #include <logging.h> @@ -257,40 +258,56 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) m_dirty_fileinfo.insert(fileNumber); } -void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) +void BlockManager::FindFilesToPruneManual( + std::set<int>& setFilesToPrune, + int nManualPruneHeight, + const Chainstate& chain, + ChainstateManager& chainman) { assert(IsPruneMode() && nManualPruneHeight > 0); LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0) { + if (chain.m_chain.Height() < 0) { return; } - // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) - unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); + const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, nManualPruneHeight); + int count = 0; - for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { - if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) { + const auto& fileinfo = m_blockfile_info[fileNumber]; + if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) { continue; } + PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; } - LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); + LogPrintf("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", + chain.GetRole(), last_block_can_prune, count); } -void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) +void BlockManager::FindFilesToPrune( + std::set<int>& setFilesToPrune, + int last_prune, + const Chainstate& chain, + ChainstateManager& chainman) { LOCK2(cs_main, cs_LastBlockFile); - if (chain_tip_height < 0 || GetPruneTarget() == 0) { + // Distribute our -prune budget over all chainstates. + const auto target = std::max( + MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget() / chainman.GetAll().size()); + + if (chain.m_chain.Height() < 0 || target == 0) { return; } - if ((uint64_t)chain_tip_height <= nPruneAfterHeight) { + if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) { return; } - unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast<int>(MIN_BLOCKS_TO_KEEP))}; + const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, last_prune); + uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -299,29 +316,31 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr uint64_t nBytesToPrune; int count = 0; - if (nCurrentUsage + nBuffer >= GetPruneTarget()) { + if (nCurrentUsage + nBuffer >= target) { // On a prune event, the chainstate DB is flushed. // To avoid excessive prune events negating the benefit of high dbcache // values, we should not prune too rapidly. // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. - if (is_ibd) { + if (chainman.IsInitialBlockDownload()) { // Since this is only relevant during IBD, we use a fixed 10% - nBuffer += GetPruneTarget() / 10; + nBuffer += target / 10; } - for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { - nBytesToPrune = m_blockfile_info[fileNumber].nSize + m_blockfile_info[fileNumber].nUndoSize; + for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) { + const auto& fileinfo = m_blockfile_info[fileNumber]; + nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize; - if (m_blockfile_info[fileNumber].nSize == 0) { + if (fileinfo.nSize == 0) { continue; } - if (nCurrentUsage + nBuffer < GetPruneTarget()) { // are we below our target? + if (nCurrentUsage + nBuffer < target) { // are we below our target? break; } - // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning - if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + // don't prune files that could have a block that's not within the allowable + // prune range for the chain being pruned. + if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) { continue; } @@ -333,10 +352,10 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr } } - LogPrint(BCLog::PRUNE, "target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", - GetPruneTarget() / 1024 / 1024, nCurrentUsage / 1024 / 1024, - (int64_t(GetPruneTarget()) - int64_t(nCurrentUsage)) / 1024 / 1024, - nLastBlockWeCanPrune, count); + LogPrint(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n", + chain.GetRole(), target / 1024 / 1024, nCurrentUsage / 1024 / 1024, + (int64_t(target) - int64_t(nCurrentUsage)) / 1024 / 1024, + min_block_to_prune, last_block_can_prune, count); } void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) { @@ -360,13 +379,32 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return pindex; } -bool BlockManager::LoadBlockIndex() +bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash) { if (!m_block_tree_db->LoadBlockIndexGuts( GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) { return false; } + if (snapshot_blockhash) { + const AssumeutxoData au_data = *Assert(GetParams().AssumeutxoForBlockhash(*snapshot_blockhash)); + m_snapshot_height = au_data.height; + CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)}; + + // Since nChainTx (responsible for estimated progress) isn't persisted + // to disk, we must bootstrap the value for assumedvalid chainstates + // from the hardcoded assumeutxo chainparams. + base->nChainTx = au_data.nChainTx; + LogPrintf("[snapshot] set nChainTx=%d for %s\n", au_data.nChainTx, snapshot_blockhash->ToString()); + } else { + // If this isn't called with a snapshot blockhash, make sure the cached snapshot height + // is null. This is relevant during snapshot completion, when the blockman may be loaded + // with a height that then needs to be cleared after the snapshot is fully validated. + m_snapshot_height.reset(); + } + + Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value()); + // Calculate nChainWork std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()}; std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), @@ -383,7 +421,11 @@ bool BlockManager::LoadBlockIndex() // Pruned nodes may have deleted the block. if (pindex->nTx > 0) { if (pindex->pprev) { - if (pindex->pprev->nChainTx > 0) { + if (m_snapshot_height && pindex->nHeight == *m_snapshot_height && + pindex->GetBlockHash() == *snapshot_blockhash) { + // Should have been set above; don't disturb it with code below. + Assert(pindex->nChainTx > 0); + } else if (pindex->pprev->nChainTx > 0) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; } else { pindex->nChainTx = 0; @@ -420,27 +462,29 @@ bool BlockManager::WriteBlockIndexDB() vBlocks.push_back(*it); m_dirty_blockindex.erase(it++); } - if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) { + int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum()); + if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks)) { return false; } return true; } -bool BlockManager::LoadBlockIndexDB() +bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash) { - if (!LoadBlockIndex()) { + if (!LoadBlockIndex(snapshot_blockhash)) { return false; } + int max_blockfile_num{0}; // Load block file info - m_block_tree_db->ReadLastBlockFile(m_last_blockfile); - m_blockfile_info.resize(m_last_blockfile + 1); - LogPrintf("%s: last block file = %i\n", __func__, m_last_blockfile); - for (int nFile = 0; nFile <= m_last_blockfile; nFile++) { + m_block_tree_db->ReadLastBlockFile(max_blockfile_num); + m_blockfile_info.resize(max_blockfile_num + 1); + LogPrintf("%s: last block file = %i\n", __func__, max_blockfile_num); + for (int nFile = 0; nFile <= max_blockfile_num; nFile++) { m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]); } - LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[m_last_blockfile].ToString()); - for (int nFile = m_last_blockfile + 1; true; nFile++) { + LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[max_blockfile_num].ToString()); + for (int nFile = max_blockfile_num + 1; true; nFile++) { CBlockFileInfo info; if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { m_blockfile_info.push_back(info); @@ -464,6 +508,15 @@ bool BlockManager::LoadBlockIndexDB() } } + { + // Initialize the blockfile cursors. + LOCK(cs_LastBlockFile); + for (size_t i = 0; i < m_blockfile_info.size(); ++i) { + const auto last_height_in_file = m_blockfile_info[i].nHeightLast; + m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0}; + } + } + // Check whether we have ever pruned block & undo files m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned); if (m_have_pruned) { @@ -481,12 +534,13 @@ bool BlockManager::LoadBlockIndexDB() void BlockManager::ScanAndUnlinkAlreadyPrunedFiles() { AssertLockHeld(::cs_main); + int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum()); if (!m_have_pruned) { return; } std::set<int> block_files_to_prune; - for (int file_number = 0; file_number < m_last_blockfile; file_number++) { + for (int file_number = 0; file_number < max_blockfile; file_number++) { if (m_blockfile_info[file_number].nSize == 0) { block_files_to_prune.insert(file_number); } @@ -651,16 +705,19 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in return true; } -void BlockManager::FlushUndoFile(int block_file, bool finalize) +bool BlockManager::FlushUndoFile(int block_file, bool finalize) { FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { m_opts.notifications.flushError("Flushing undo file to disk failed. This is likely the result of an I/O error."); + return false; } + return true; } -void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) +bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo) { + bool success = true; LOCK(cs_LastBlockFile); if (m_blockfile_info.size() < 1) { @@ -668,17 +725,43 @@ void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which // then calls FlushStateToDisk()), resulting in a call to this function before we // have populated `m_blockfile_info` via LoadBlockIndexDB(). - return; + return true; } - assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile); + assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num); - FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); + FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { m_opts.notifications.flushError("Flushing block file to disk failed. This is likely the result of an I/O error."); + success = false; } // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(m_last_blockfile, finalize_undo); + if (!fFinalize || finalize_undo) { + if (!FlushUndoFile(blockfile_num, finalize_undo)) { + success = false; + } + } + return success; +} + +BlockfileType BlockManager::BlockfileTypeForHeight(int height) +{ + if (!m_snapshot_height) { + return BlockfileType::NORMAL; + } + return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL; +} + +bool BlockManager::FlushChainstateBlockFile(int tip_height) +{ + LOCK(cs_LastBlockFile); + auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)]; + if (cursor) { + // The cursor may not exist after a snapshot has been loaded but before any + // blocks have been downloaded. + return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false); + } + return false; } uint64_t BlockManager::CalculateCurrentUsage() @@ -735,8 +818,19 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne { LOCK(cs_LastBlockFile); - unsigned int nFile = fKnown ? pos.nFile : m_last_blockfile; - if (m_blockfile_info.size() <= nFile) { + const BlockfileType chain_type = BlockfileTypeForHeight(nHeight); + + if (!m_blockfile_cursors[chain_type]) { + // If a snapshot is loaded during runtime, we may not have initialized this cursor yet. + assert(chain_type == BlockfileType::ASSUMED); + const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1}; + m_blockfile_cursors[chain_type] = new_cursor; + LogPrint(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor); + } + const int last_blockfile = m_blockfile_cursors[chain_type]->file_num; + + int nFile = fKnown ? pos.nFile : last_blockfile; + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { m_blockfile_info.resize(nFile + 1); } @@ -753,13 +847,20 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne } } assert(nAddSize < max_blockfile_size); + while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it - finalize_undo = (m_blockfile_info[nFile].nHeightLast == m_undo_height_in_last_blockfile); - nFile++; - if (m_blockfile_info.size() <= nFile) { + finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) == + Assert(m_blockfile_cursors[chain_type])->undo_height); + + // Try the next unclaimed blockfile number + nFile = this->MaxBlockfileNum() + 1; + // Set to increment MaxBlockfileNum() for next iteration + m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; + + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { m_blockfile_info.resize(nFile + 1); } } @@ -767,13 +868,26 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne pos.nPos = m_blockfile_info[nFile].nSize; } - if ((int)nFile != m_last_blockfile) { + if (nFile != last_blockfile) { if (!fKnown) { - LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString()); + LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", + last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); + } + + // Do not propagate the return code. The flush concerns a previous block + // and undo file that has already been written to. If a flush fails + // here, and we crash, there is no expected additional block data + // inconsistency arising from the flush failure here. However, the undo + // data may be inconsistent after a crash if the flush is called during + // a reindex. A flush error might also leave some of the data files + // untrimmed. + if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) { + LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, + "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n", + last_blockfile, !fKnown, finalize_undo, nFile); } - FlushBlockFile(!fKnown, finalize_undo); - m_last_blockfile = nFile; - m_undo_height_in_last_blockfile = 0; // No undo data yet in the new file, so reset our undo-height tracking. + // No undo data yet in the new file, so reset our undo-height tracking. + m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; } m_blockfile_info[nFile].AddBlock(nHeight, nTime); @@ -847,6 +961,9 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); + const BlockfileType type = BlockfileTypeForHeight(block.nHeight); + auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type])); + // Write undo information to disk if (block.GetUndoPos().IsNull()) { FlatFilePos _pos; @@ -861,10 +978,17 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in // the FindBlockPos function - if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { - FlushUndoFile(_pos.nFile, true); - } else if (_pos.nFile == m_last_blockfile && static_cast<uint32_t>(block.nHeight) > m_undo_height_in_last_blockfile) { - m_undo_height_in_last_blockfile = block.nHeight; + if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { + // Do not propagate the return code, a failed flush here should not + // be an indication for a failed write. If it were propagated here, + // the caller would assume the undo data not to be written, when in + // fact it is. Note though, that a failed flush might leave the data + // file untrimmed. + if (!FlushUndoFile(_pos.nFile, true)) { + LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", _pos.nFile); + } + } else if (_pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) { + cursor.undo_height = block.nHeight; } // update nUndoPos in block index block.nUndoPos = _pos.nPos; @@ -1063,4 +1187,18 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile } } // End scope of ImportingNow } + +std::ostream& operator<<(std::ostream& os, const BlockfileType& type) { + switch(type) { + case BlockfileType::NORMAL: os << "normal"; break; + case BlockfileType::ASSUMED: os << "assumed"; break; + default: os.setstate(std::ios_base::failbit); + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) { + os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height); + return os; +} } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index a89fa7f76e..ac97728c05 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -9,6 +9,7 @@ #include <chain.h> #include <dbwrapper.h> #include <kernel/blockmanager_opts.h> +#include <kernel/chain.h> #include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <kernel/messagestartchars.h> @@ -97,6 +98,35 @@ struct PruneLockInfo { int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned }; +enum BlockfileType { + // Values used as array indexes - do not change carelessly. + NORMAL = 0, + ASSUMED = 1, + NUM_TYPES = 2, +}; + +std::ostream& operator<<(std::ostream& os, const BlockfileType& type); + +struct BlockfileCursor { + // The latest blockfile number. + int file_num{0}; + + // Track the height of the highest block in file_num whose undo + // data has been written. Block data is written to block files in download + // order, but is written to undo files in validation order, which is + // usually in order by height. To avoid wasting disk space, undo files will + // be trimmed whenever the corresponding block file is finalized and + // the height of the highest block written to the block file equals the + // height of the highest block written to the undo file. This is a + // heuristic and can sometimes preemptively trim undo files that will write + // more data later, and sometimes fail to trim undo files that can't have + // more data written later. + int undo_height{0}; +}; + +std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor); + + /** * Maintains a tree of blocks (stored in `m_block_index`) which is consulted * to determine where the most-work tip is. @@ -117,11 +147,17 @@ private: * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like m_dirty_blockindex. */ - bool LoadBlockIndex() + bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); - void FlushUndoFile(int block_file, bool finalize = false); - bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); + + /** Return false if block file or undo file flushing fails. */ + [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo); + + /** Return false if undo file flushing fails. */ + [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false); + + [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); + [[nodiscard]] bool FlushChainstateBlockFile(int tip_height); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); FlatFileSeq BlockFileSeq() const; @@ -133,7 +169,11 @@ private: bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ - void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); + void FindFilesToPruneManual( + std::set<int>& setFilesToPrune, + int nManualPruneHeight, + const Chainstate& chain, + ChainstateManager& chainman); /** * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target. @@ -149,24 +189,39 @@ private: * A db flag records the fact that at least some block files have been pruned. * * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + * @param last_prune The last height we're able to prune, according to the prune locks */ - void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); + void FindFilesToPrune( + std::set<int>& setFilesToPrune, + int last_prune, + const Chainstate& chain, + ChainstateManager& chainman); RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> m_blockfile_info; - int m_last_blockfile = 0; - // Track the height of the highest block in m_last_blockfile whose undo - // data has been written. Block data is written to block files in download - // order, but is written to undo files in validation order, which is - // usually in order by height. To avoid wasting disk space, undo files will - // be trimmed whenever the corresponding block file is finalized and - // the height of the highest block written to the block file equals the - // height of the highest block written to the undo file. This is a - // heuristic and can sometimes preemptively trim undo files that will write - // more data later, and sometimes fail to trim undo files that can't have - // more data written later. - unsigned int m_undo_height_in_last_blockfile = 0; + //! Since assumedvalid chainstates may be syncing a range of the chain that is very + //! far away from the normal/background validation process, we should segment blockfiles + //! for assumed chainstates. Otherwise, we might have wildly different height ranges + //! mixed into the same block files, which would impair our ability to prune + //! effectively. + //! + //! This data structure maintains separate blockfile number cursors for each + //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindBlockPos(). + //! + //! The first element is the NORMAL cursor, second is ASSUMED. + std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES> + m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = { + BlockfileCursor{}, + std::nullopt, + }; + int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile) + { + static const BlockfileCursor empty_cursor; + const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor); + const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor); + return std::max(normal.file_num, assumed.file_num); + } /** Global flag to indicate we should check to see if there are * block/undo files that should be deleted. Set on startup @@ -190,6 +245,8 @@ private: */ std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main); + BlockfileType BlockfileTypeForHeight(int height); + const kernel::BlockManagerOpts m_opts; public: @@ -205,6 +262,20 @@ public: BlockMap m_block_index GUARDED_BY(cs_main); + /** + * The height of the base block of an assumeutxo snapshot, if one is in use. + * + * This controls how blockfiles are segmented by chainstate type to avoid + * comingling different height regions of the chain when an assumedvalid chainstate + * is in use. If heights are drastically different in the same blockfile, pruning + * suffers. + * + * This is set during ActivateSnapshot() or upon LoadBlockIndex() if a snapshot + * had been previously loaded. After the snapshot is validated, this is unset to + * restore normal LoadBlockIndex behavior. + */ + std::optional<int> m_snapshot_height; + std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** @@ -216,7 +287,8 @@ public: std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Remove any pruned block & undo files that are still on disk. diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index ae1457a87e..16ca1d9156 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -185,7 +185,14 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize chainman.InitializeChainstate(options.mempool); // Load a chain created from a UTXO snapshot, if any exist. - chainman.DetectSnapshotChainstate(options.mempool); + bool has_snapshot = chainman.DetectSnapshotChainstate(options.mempool); + + if (has_snapshot && (options.reindex || options.reindex_chainstate)) { + LogPrintf("[snapshot] deleting snapshot chainstate due to reindexing\n"); + if (!chainman.DeleteSnapshotChainstate()) { + return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; + } + } auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); if (init_status != ChainstateLoadStatus::SUCCESS) { diff --git a/src/node/connection_types.cpp b/src/node/connection_types.cpp index 904f4371aa..5e4dc5bf2e 100644 --- a/src/node/connection_types.cpp +++ b/src/node/connection_types.cpp @@ -24,3 +24,17 @@ std::string ConnectionTypeAsString(ConnectionType conn_type) assert(false); } + +std::string TransportTypeAsString(TransportProtocolType transport_type) +{ + switch (transport_type) { + case TransportProtocolType::DETECTING: + return "detecting"; + case TransportProtocolType::V1: + return "v1"; + case TransportProtocolType::V2: + return "v2"; + } // no default case, so the compiler can warn about missing cases + + assert(false); +} diff --git a/src/node/connection_types.h b/src/node/connection_types.h index 5e1abcace6..a911b95f7e 100644 --- a/src/node/connection_types.h +++ b/src/node/connection_types.h @@ -6,6 +6,7 @@ #define BITCOIN_NODE_CONNECTION_TYPES_H #include <string> +#include <stdint.h> /** Different types of connections to a peer. This enum encapsulates the * information we have available at the time of opening or accepting the @@ -79,4 +80,14 @@ enum class ConnectionType { /** Convert ConnectionType enum to a string value */ std::string ConnectionTypeAsString(ConnectionType conn_type); +/** Transport layer version */ +enum class TransportProtocolType : uint8_t { + DETECTING, //!< Peer could be v1 or v2 + V1, //!< Unencrypted, plaintext protocol + V2, //!< BIP324 protocol +}; + +/** Convert TransportProtocolType enum to a string value */ +std::string TransportTypeAsString(TransportProtocolType transport_type); + #endif // BITCOIN_NODE_CONNECTION_TYPES_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index e0c40036d9..4baa0da67c 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -434,9 +434,9 @@ public: { m_notifications->transactionRemovedFromMempool(tx, reason); } - void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override + void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->blockConnected(kernel::MakeBlockInfo(index, block.get())); + m_notifications->blockConnected(role, kernel::MakeBlockInfo(index, block.get())); } void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { @@ -446,7 +446,9 @@ public: { m_notifications->updatedBlockTip(); } - void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->chainStateFlushed(locator); } + void ChainStateFlushed(ChainstateRole role, const CBlockLocator& locator) override { + m_notifications->chainStateFlushed(role, locator); + } std::shared_ptr<Chain::Notifications> m_notifications; }; diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 50a30cb511..3d21708820 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -10,7 +10,7 @@ uint256 CBlockHeader::GetHash() const { - return SerializeHash(*this); + return (CHashWriter{PROTOCOL_VERSION} << *this).GetHash(); } std::string CBlock::ToString() const diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 3060746909..2c913bf432 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -67,12 +67,12 @@ CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), uint256 CMutableTransaction::GetHash() const { - return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); + return (CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash(); } uint256 CTransaction::ComputeHash() const { - return SerializeHash(*this, SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); + return (CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash(); } uint256 CTransaction::ComputeWitnessHash() const @@ -80,7 +80,7 @@ uint256 CTransaction::ComputeWitnessHash() const if (!HasWitness()) { return hash; } - return SerializeHash(*this, SER_GETHASH, 0); + return (CHashWriter{0} << *this).GetHash(); } CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} diff --git a/src/protocol.cpp b/src/protocol.cpp index cb956191e4..f956728af2 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -199,6 +199,7 @@ static std::string serviceFlagToStr(size_t bit) case NODE_WITNESS: return "WITNESS"; case NODE_COMPACT_FILTERS: return "COMPACT_FILTERS"; case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED"; + case NODE_P2P_V2: return "P2P_V2"; // Not using default, so we get warned when a case is missing } diff --git a/src/protocol.h b/src/protocol.h index 56668898e4..a58d671a70 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -291,6 +291,9 @@ enum ServiceFlags : uint64_t { // See BIP159 for details on how this is implemented. NODE_NETWORK_LIMITED = (1 << 10), + // NODE_P2P_V2 means the node supports BIP324 transport + NODE_P2P_V2 = (1 << 11), + // Bits 24-31 are reserved for temporary experiments. Just pick a bit that // isn't getting used, or one not being used much, and notify the // bitcoin-development mailing list. Remember that service bits are just diff --git a/src/psbt.h b/src/psbt.h index 9464b10268..48e0453084 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -226,7 +226,7 @@ struct PSBTInput // Write the utxo if (non_witness_utxo) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_NON_WITNESS_UTXO)); - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; SerializeToVector(os, non_witness_utxo); } if (!witness_utxo.IsNull()) { @@ -315,7 +315,7 @@ struct PSBTInput const auto& [leaf_hashes, origin] = leaf_origin; SerializeToVector(s, PSBT_IN_TAP_BIP32_DERIVATION, xonly); std::vector<unsigned char> value; - CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); + CVectorWriter s_value{s.GetVersion(), value, 0}; s_value << leaf_hashes; SerializeKeyOrigin(s_value, origin); s << value; @@ -381,7 +381,7 @@ struct PSBTInput } // Type is compact size uint at beginning of key - SpanReader skey(s.GetType(), s.GetVersion(), key); + SpanReader skey{s.GetVersion(), key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -394,7 +394,7 @@ struct PSBTInput throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); } // Set the stream to unserialize with witness since this is always a valid network transaction - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS); + OverrideStream<Stream> os{&s, s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS}; UnserializeFromVector(os, non_witness_utxo); break; } @@ -590,7 +590,7 @@ struct PSBTInput } else if (key.size() != 65) { throw std::ios_base::failure("Input Taproot script signature key is not 65 bytes"); } - SpanReader s_key(s.GetType(), s.GetVersion(), Span{key}.subspan(1)); + SpanReader s_key{s.GetVersion(), Span{key}.subspan(1)}; XOnlyPubKey xonly; uint256 hash; s_key >> xonly; @@ -632,7 +632,7 @@ struct PSBTInput } else if (key.size() != 33) { throw std::ios_base::failure("Input Taproot BIP32 keypath key is not at 33 bytes"); } - SpanReader s_key(s.GetType(), s.GetVersion(), Span{key}.subspan(1)); + SpanReader s_key{s.GetVersion(), Span{key}.subspan(1)}; XOnlyPubKey xonly; s_key >> xonly; std::set<uint256> leaf_hashes; @@ -757,7 +757,7 @@ struct PSBTOutput if (!m_tap_tree.empty()) { SerializeToVector(s, PSBT_OUT_TAP_TREE); std::vector<unsigned char> value; - CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); + CVectorWriter s_value{s.GetVersion(), value, 0}; for (const auto& [depth, leaf_ver, script] : m_tap_tree) { s_value << depth; s_value << leaf_ver; @@ -771,7 +771,7 @@ struct PSBTOutput const auto& [leaf_hashes, origin] = leaf; SerializeToVector(s, PSBT_OUT_TAP_BIP32_DERIVATION, xonly); std::vector<unsigned char> value; - CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); + CVectorWriter s_value{s.GetVersion(), value, 0}; s_value << leaf_hashes; SerializeKeyOrigin(s_value, origin); s << value; @@ -807,7 +807,7 @@ struct PSBTOutput } // Type is compact size uint at beginning of key - SpanReader skey(s.GetType(), s.GetVersion(), key); + SpanReader skey{s.GetVersion(), key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -856,7 +856,7 @@ struct PSBTOutput } std::vector<unsigned char> tree_v; s >> tree_v; - SpanReader s_tree(s.GetType(), s.GetVersion(), tree_v); + SpanReader s_tree{s.GetVersion(), tree_v}; if (s_tree.empty()) { throw std::ios_base::failure("Output Taproot tree must not be empty"); } @@ -984,7 +984,7 @@ struct PartiallySignedTransaction SerializeToVector(s, CompactSizeWriter(PSBT_GLOBAL_UNSIGNED_TX)); // Write serialized tx to a stream - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; SerializeToVector(os, *tx); // Write xpubs @@ -1061,7 +1061,7 @@ struct PartiallySignedTransaction } // Type is compact size uint at beginning of key - SpanReader skey(s.GetType(), s.GetVersion(), key); + SpanReader skey{s.GetVersion(), key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -1075,7 +1075,7 @@ struct PartiallySignedTransaction } CMutableTransaction mtx; // Set the stream to serialize with non-witness since this should always be non-witness - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; UnserializeFromVector(os, mtx); tx = std::move(mtx); // Make sure that all scriptSigs and scriptWitnesses are empty diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f4d88e4209..0f4941b40c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -8,6 +8,7 @@ #include <blockfilter.h> #include <chain.h> #include <chainparams.h> +#include <clientversion.h> #include <coins.h> #include <common/args.h> #include <consensus/amount.h> @@ -2699,6 +2700,178 @@ UniValue CreateUTXOSnapshot( return result; } +static RPCHelpMan loadtxoutset() +{ + return RPCHelpMan{ + "loadtxoutset", + "Load the serialized UTXO set from disk.\n" + "Once this snapshot is loaded, its contents will be " + "deserialized into a second chainstate data structure, which is then used to sync to " + "the network's tip under a security model very much like `assumevalid`. " + "Meanwhile, the original chainstate will complete the initial block download process in " + "the background, eventually validating up to the block that the snapshot is based upon.\n\n" + + "The result is a usable bitcoind instance that is current with the network tip in a " + "matter of minutes rather than hours. UTXO snapshot are typically obtained from " + "third-party sources (HTTP, torrent, etc.) which is reasonable since their " + "contents are always checked by hash.\n\n" + + "You can find more information on this process in the `assumeutxo` design " + "document (<https://github.com/bitcoin/bitcoin/blob/master/doc/design/assumeutxo.md>).", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + "path to the snapshot file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "coins_loaded", "the number of coins loaded from the snapshot"}, + {RPCResult::Type::STR_HEX, "tip_hash", "the hash of the base of the snapshot"}, + {RPCResult::Type::NUM, "base_height", "the height of the base of the snapshot"}, + {RPCResult::Type::STR, "path", "the absolute path that the snapshot was loaded from"}, + } + }, + RPCExamples{ + HelpExampleCli("loadtxoutset", "utxo.dat") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + NodeContext& node = EnsureAnyNodeContext(request.context); + fs::path path{AbsPathForConfigVal(EnsureArgsman(node), fs::u8path(request.params[0].get_str()))}; + + FILE* file{fsbridge::fopen(path, "rb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + "Couldn't open file " + path.u8string() + " for reading."); + } + + SnapshotMetadata metadata; + afile >> metadata; + + uint256 base_blockhash = metadata.m_base_blockhash; + int max_secs_to_wait_for_headers = 60 * 10; + CBlockIndex* snapshot_start_block = nullptr; + + LogPrintf("[snapshot] waiting to see blockheader %s in headers chain before snapshot activation\n", + base_blockhash.ToString()); + + ChainstateManager& chainman = *node.chainman; + + while (max_secs_to_wait_for_headers > 0) { + snapshot_start_block = WITH_LOCK(::cs_main, + return chainman.m_blockman.LookupBlockIndex(base_blockhash)); + max_secs_to_wait_for_headers -= 1; + + if (!IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + + if (!snapshot_start_block) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } else { + break; + } + } + + if (!snapshot_start_block) { + LogPrintf("[snapshot] timed out waiting for snapshot start blockheader %s\n", + base_blockhash.ToString()); + throw JSONRPCError( + RPC_INTERNAL_ERROR, + "Timed out waiting for base block header to appear in headers chain"); + } + if (!chainman.ActivateSnapshot(afile, metadata, false)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to load UTXO snapshot " + fs::PathToString(path)); + } + CBlockIndex* new_tip{WITH_LOCK(::cs_main, return chainman.ActiveTip())}; + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_loaded", metadata.m_coins_count); + result.pushKV("tip_hash", new_tip->GetBlockHash().ToString()); + result.pushKV("base_height", new_tip->nHeight); + result.pushKV("path", fs::PathToString(path)); + return result; +}, + }; +} + +const std::vector<RPCResult> RPCHelpForChainstate{ + {RPCResult::Type::NUM, "blocks", "number of blocks in this chainstate"}, + {RPCResult::Type::STR_HEX, "bestblockhash", "blockhash of the tip"}, + {RPCResult::Type::NUM, "difficulty", "difficulty of the tip"}, + {RPCResult::Type::NUM, "verificationprogress", "progress towards the network tip"}, + {RPCResult::Type::STR_HEX, "snapshot_blockhash", /*optional=*/true, "the base block of the snapshot this chainstate is based on, if any"}, + {RPCResult::Type::NUM, "coins_db_cache_bytes", "size of the coinsdb cache"}, + {RPCResult::Type::NUM, "coins_tip_cache_bytes", "size of the coinstip cache"}, +}; + +static RPCHelpMan getchainstates() +{ +return RPCHelpMan{ + "getchainstates", + "\nReturn information about chainstates.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "headers", "the number of headers seen so far"}, + {RPCResult::Type::OBJ, "normal", /*optional=*/true, "fully validated chainstate containing blocks this node has validated starting from the genesis block", RPCHelpForChainstate}, + {RPCResult::Type::OBJ, "snapshot", /*optional=*/true, "only present if an assumeutxo snapshot is loaded. Partially validated chainstate containing blocks this node has validated starting from the snapshot. After the snapshot is validated (when the 'normal' chainstate advances far enough to validate it), this chainstate will replace and become the 'normal' chainstate.", RPCHelpForChainstate}, + } + }, + RPCExamples{ + HelpExampleCli("getchainstates", "") + + HelpExampleRpc("getchainstates", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + LOCK(cs_main); + UniValue obj(UniValue::VOBJ); + + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = *node.chainman; + + auto make_chain_data = [&](const Chainstate& cs) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + AssertLockHeld(::cs_main); + UniValue data(UniValue::VOBJ); + if (!cs.m_chain.Tip()) { + return data; + } + const CChain& chain = cs.m_chain; + const CBlockIndex* tip = chain.Tip(); + + data.pushKV("blocks", (int)chain.Height()); + data.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); + data.pushKV("difficulty", (double)GetDifficulty(tip)); + data.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); + data.pushKV("coins_db_cache_bytes", cs.m_coinsdb_cache_size_bytes); + data.pushKV("coins_tip_cache_bytes", cs.m_coinstip_cache_size_bytes); + if (cs.m_from_snapshot_blockhash) { + data.pushKV("snapshot_blockhash", cs.m_from_snapshot_blockhash->ToString()); + } + return data; + }; + + if (chainman.GetAll().size() > 1) { + for (Chainstate* chainstate : chainman.GetAll()) { + obj.pushKV( + chainstate->m_from_snapshot_blockhash ? "snapshot" : "normal", + make_chain_data(*chainstate)); + } + } else { + obj.pushKV("normal", make_chain_data(chainman.ActiveChainstate())); + } + obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); + + return obj; +} + }; +} + + void RegisterBlockchainRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -2722,13 +2895,15 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &scantxoutset}, {"blockchain", &scanblocks}, {"blockchain", &getblockfilter}, + {"blockchain", &dumptxoutset}, + {"blockchain", &loadtxoutset}, + {"blockchain", &getchainstates}, {"hidden", &invalidateblock}, {"hidden", &reconsiderblock}, {"hidden", &waitfornewblock}, {"hidden", &waitforblock}, {"hidden", &waitforblockheight}, {"hidden", &syncwithvalidationinterfacequeue}, - {"hidden", &dumptxoutset}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 0ee3f27761..49820f25a3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -263,13 +263,13 @@ static const CRPCConvertParam vRPCConvertParams[] = { "bumpfee", 1, "fee_rate"}, { "bumpfee", 1, "replaceable"}, { "bumpfee", 1, "outputs"}, - { "bumpfee", 1, "reduce_output"}, + { "bumpfee", 1, "original_change_index"}, { "psbtbumpfee", 1, "options" }, { "psbtbumpfee", 1, "conf_target"}, { "psbtbumpfee", 1, "fee_rate"}, { "psbtbumpfee", 1, "replaceable"}, { "psbtbumpfee", 1, "outputs"}, - { "psbtbumpfee", 1, "reduce_output"}, + { "psbtbumpfee", 1, "original_change_index"}, { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, @@ -301,6 +301,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "addpeeraddress", 2, "tried"}, { "sendmsgtopeer", 0, "peer_id" }, { "stop", 0, "wait" }, + { "addnode", 2, "v2transport" }, }; // clang-format on diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 6af62641bd..8d796b8e9b 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -45,6 +45,12 @@ const std::vector<std::string> CONNECTION_TYPE_DOC{ "feeler (short-lived automatic connection for testing addresses)" }; +const std::vector<std::string> TRANSPORT_TYPE_DOC{ + "detecting (peer could be v1 or v2)", + "v1 (plaintext transport protocol)", + "v2 (BIP324 encrypted transport protocol)" +}; + static RPCHelpMan getconnectioncount() { return RPCHelpMan{"getconnectioncount", @@ -164,6 +170,8 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" "best capture connection behaviors."}, + {RPCResult::Type::STR, "transport_protocol_type", "Type of transport protocol: \n" + Join(TRANSPORT_TYPE_DOC, ",\n") + ".\n"}, + {RPCResult::Type::STR, "session_id", "The session ID for this connection, or \"\" if there is none (\"v2\" transport protocol only).\n"}, }}, }}, }, @@ -268,6 +276,8 @@ static RPCHelpMan getpeerinfo() } obj.pushKV("bytesrecv_per_msg", recvPerMsgType); obj.pushKV("connection_type", ConnectionTypeAsString(stats.m_conn_type)); + obj.pushKV("transport_protocol_type", TransportTypeAsString(stats.m_transport_type)); + obj.pushKV("session_id", stats.m_session_id); ret.push_back(obj); } @@ -289,11 +299,12 @@ static RPCHelpMan addnode() { {"node", RPCArg::Type::STR, RPCArg::Optional::NO, "The address of the peer to connect to"}, {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once"}, + {"v2transport", RPCArg::Type::BOOL, RPCArg::Default{false}, "Attempt to connect using BIP324 v2 transport protocol (ignored for 'remove' command)"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ - HelpExampleCli("addnode", "\"192.168.0.6:8333\" \"onetry\"") - + HelpExampleRpc("addnode", "\"192.168.0.6:8333\", \"onetry\"") + HelpExampleCli("addnode", "\"192.168.0.6:8333\" \"onetry\" true") + + HelpExampleRpc("addnode", "\"192.168.0.6:8333\", \"onetry\" true") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -307,17 +318,22 @@ static RPCHelpMan addnode() CConnman& connman = EnsureConnman(node); const std::string node_arg{request.params[0].get_str()}; + bool use_v2transport = self.Arg<bool>(2); + + if (use_v2transport && !(node.connman->GetLocalServices() & NODE_P2P_V2)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: v2transport requested but not enabled (see -v2transport)"); + } if (command == "onetry") { CAddress addr; - connman.OpenNetworkConnection(addr, /*fCountFailure=*/false, /*grantOutbound=*/nullptr, node_arg.c_str(), ConnectionType::MANUAL); + connman.OpenNetworkConnection(addr, /*fCountFailure=*/false, /*grant_outbound=*/{}, node_arg.c_str(), ConnectionType::MANUAL, use_v2transport); return UniValue::VNULL; } if (command == "add") { - if (!connman.AddNode(node_arg)) { + if (!connman.AddNode({node_arg, use_v2transport})) { throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Node already added"); } } @@ -475,7 +491,7 @@ static RPCHelpMan getaddednodeinfo() if (!request.params[0].isNull()) { bool found = false; for (const AddedNodeInfo& info : vInfo) { - if (info.strAddedNode == request.params[0].get_str()) { + if (info.m_params.m_added_node == request.params[0].get_str()) { vInfo.assign(1, info); found = true; break; @@ -490,7 +506,7 @@ static RPCHelpMan getaddednodeinfo() for (const AddedNodeInfo& info : vInfo) { UniValue obj(UniValue::VOBJ); - obj.pushKV("addednode", info.strAddedNode); + obj.pushKV("addednode", info.m_params.m_added_node); obj.pushKV("connected", info.fConnected); UniValue addresses(UniValue::VARR); if (info.fConnected) { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 9a941be181..a11366bd47 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -682,6 +682,7 @@ TMPL_INST(nullptr, std::optional<bool>, maybe_arg ? std::optional{maybe_arg->get TMPL_INST(nullptr, const std::string*, maybe_arg ? &maybe_arg->get_str() : nullptr;); // Required arg or optional arg with default value. +TMPL_INST(CheckRequiredOrDefault, bool, CHECK_NONFATAL(maybe_arg)->get_bool();); TMPL_INST(CheckRequiredOrDefault, int, CHECK_NONFATAL(maybe_arg)->getInt<int>();); TMPL_INST(CheckRequiredOrDefault, uint64_t, CHECK_NONFATAL(maybe_arg)->getInt<uint64_t>();); TMPL_INST(CheckRequiredOrDefault, const std::string&, CHECK_NONFATAL(maybe_arg)->get_str();); diff --git a/src/serialize.h b/src/serialize.h index 1ad8ac4373..e53ff9fa4c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -131,7 +131,6 @@ enum // primary actions SER_NETWORK = (1 << 0), SER_DISK = (1 << 1), - SER_GETHASH = (1 << 2), }; /** diff --git a/src/signet.cpp b/src/signet.cpp index 21b289b637..ef0faaa5f8 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -98,7 +98,7 @@ std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& c // no signet solution -- allow this to support OP_TRUE as trivial block challenge } else { try { - SpanReader v{SER_NETWORK, INIT_PROTO_VERSION, signet_solution}; + SpanReader v{INIT_PROTO_VERSION, signet_solution}; v >> tx_spending.vin[0].scriptSig; v >> tx_spending.vin[0].scriptWitness.stack; if (!v.empty()) return std::nullopt; // extraneous data encountered @@ -109,7 +109,7 @@ std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& c uint256 signet_merkle = ComputeModifiedMerkleRoot(modified_cb, block); std::vector<uint8_t> block_data; - CVectorWriter writer(SER_NETWORK, INIT_PROTO_VERSION, block_data, 0); + CVectorWriter writer{INIT_PROTO_VERSION, block_data, 0}; writer << block.nVersion; writer << block.hashPrevBlock; writer << signet_merkle; diff --git a/src/streams.h b/src/streams.h index e069719d89..d58de5233b 100644 --- a/src/streams.h +++ b/src/streams.h @@ -50,11 +50,10 @@ class OverrideStream { Stream* stream; - const int nType; const int nVersion; public: - OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} + OverrideStream(Stream* stream_, int nVersion_) : stream{stream_}, nVersion{nVersion_} {} template<typename T> OverrideStream<Stream>& operator<<(const T& obj) @@ -81,7 +80,6 @@ public: } int GetVersion() const { return nVersion; } - int GetType() const { return nType; } size_t size() const { return stream->size(); } void ignore(size_t size) { return stream->ignore(size); } }; @@ -95,13 +93,12 @@ class CVectorWriter public: /* - * @param[in] nTypeIn Serialization Type * @param[in] nVersionIn Serialization Version (including any flags) * @param[in] vchDataIn Referenced byte vector to overwrite/append * @param[in] nPosIn Starting position. Vector index where writes should start. The vector will initially * grow as necessary to max(nPosIn, vec.size()). So to append, use vec.size(). */ - CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn) + CVectorWriter(int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nVersion{nVersionIn}, vchData{vchDataIn}, nPos{nPosIn} { if(nPos > vchData.size()) vchData.resize(nPos); @@ -111,7 +108,7 @@ class CVectorWriter * @param[in] args A list of items to serialize starting at nPosIn. */ template <typename... Args> - CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn) + CVectorWriter(int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter{nVersionIn, vchDataIn, nPosIn} { ::SerializeMany(*this, std::forward<Args>(args)...); } @@ -137,12 +134,8 @@ class CVectorWriter { return nVersion; } - int GetType() const - { - return nType; - } + private: - const int nType; const int nVersion; std::vector<unsigned char>& vchData; size_t nPos; @@ -153,19 +146,16 @@ private: class SpanReader { private: - const int m_type; const int m_version; Span<const unsigned char> m_data; public: - /** - * @param[in] type Serialization Type * @param[in] version Serialization Version (including any flags) * @param[in] data Referenced byte vector to overwrite/append */ - SpanReader(int type, int version, Span<const unsigned char> data) - : m_type(type), m_version(version), m_data(data) {} + SpanReader(int version, Span<const unsigned char> data) + : m_version{version}, m_data{data} {} template<typename T> SpanReader& operator>>(T&& obj) @@ -175,7 +165,6 @@ public: } int GetVersion() const { return m_version; } - int GetType() const { return m_type; } size_t size() const { return m_data.size(); } bool empty() const { return m_data.empty(); } diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 558f835f11..4395567722 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -57,4 +57,28 @@ struct secure_allocator { // TODO: Consider finding a way to make incoming RPC request.params[i] mlock()ed as well typedef std::basic_string<char, std::char_traits<char>, secure_allocator<char> > SecureString; +template<typename T> +struct SecureUniqueDeleter { + void operator()(T* t) noexcept { + secure_allocator<T>().deallocate(t, 1); + } +}; + +template<typename T> +using secure_unique_ptr = std::unique_ptr<T, SecureUniqueDeleter<T>>; + +template<typename T, typename... Args> +secure_unique_ptr<T> make_secure_unique(Args&&... as) +{ + T* p = secure_allocator<T>().allocate(1); + + // initialize in place, and return as secure_unique_ptr + try { + return secure_unique_ptr<T>(new (p) T(std::forward(as)...)); + } catch (...) { + secure_allocator<T>().deallocate(p, 1); + throw; + } +} + #endif // BITCOIN_SUPPORT_ALLOCATORS_SECURE_H diff --git a/src/sync.h b/src/sync.h index 7242a793ab..45d40b5fdc 100644 --- a/src/sync.h +++ b/src/sync.h @@ -301,6 +301,10 @@ inline MutexType* MaybeCheckNotHeld(MutexType* m) LOCKS_EXCLUDED(m) LOCK_RETURNE //! gcc and the -Wreturn-stack-address flag in clang, both enabled by default. #define WITH_LOCK(cs, code) (MaybeCheckNotHeld(cs), [&]() -> decltype(auto) { LOCK(cs); code; }()) +/** An implementation of a semaphore. + * + * See https://en.wikipedia.org/wiki/Semaphore_(programming) + */ class CSemaphore { private: @@ -309,25 +313,33 @@ private: int value; public: - explicit CSemaphore(int init) : value(init) {} + explicit CSemaphore(int init) noexcept : value(init) {} - void wait() + // Disallow default construct, copy, move. + CSemaphore() = delete; + CSemaphore(const CSemaphore&) = delete; + CSemaphore(CSemaphore&&) = delete; + CSemaphore& operator=(const CSemaphore&) = delete; + CSemaphore& operator=(CSemaphore&&) = delete; + + void wait() noexcept { std::unique_lock<std::mutex> lock(mutex); condition.wait(lock, [&]() { return value >= 1; }); value--; } - bool try_wait() + bool try_wait() noexcept { std::lock_guard<std::mutex> lock(mutex); - if (value < 1) + if (value < 1) { return false; + } value--; return true; } - void post() + void post() noexcept { { std::lock_guard<std::mutex> lock(mutex); @@ -345,45 +357,64 @@ private: bool fHaveGrant; public: - void Acquire() + void Acquire() noexcept { - if (fHaveGrant) + if (fHaveGrant) { return; + } sem->wait(); fHaveGrant = true; } - void Release() + void Release() noexcept { - if (!fHaveGrant) + if (!fHaveGrant) { return; + } sem->post(); fHaveGrant = false; } - bool TryAcquire() + bool TryAcquire() noexcept { - if (!fHaveGrant && sem->try_wait()) + if (!fHaveGrant && sem->try_wait()) { fHaveGrant = true; + } return fHaveGrant; } - void MoveTo(CSemaphoreGrant& grant) + // Disallow copy. + CSemaphoreGrant(const CSemaphoreGrant&) = delete; + CSemaphoreGrant& operator=(const CSemaphoreGrant&) = delete; + + // Allow move. + CSemaphoreGrant(CSemaphoreGrant&& other) noexcept + { + sem = other.sem; + fHaveGrant = other.fHaveGrant; + other.fHaveGrant = false; + other.sem = nullptr; + } + + CSemaphoreGrant& operator=(CSemaphoreGrant&& other) noexcept { - grant.Release(); - grant.sem = sem; - grant.fHaveGrant = fHaveGrant; - fHaveGrant = false; + Release(); + sem = other.sem; + fHaveGrant = other.fHaveGrant; + other.fHaveGrant = false; + other.sem = nullptr; + return *this; } - CSemaphoreGrant() : sem(nullptr), fHaveGrant(false) {} + CSemaphoreGrant() noexcept : sem(nullptr), fHaveGrant(false) {} - explicit CSemaphoreGrant(CSemaphore& sema, bool fTry = false) : sem(&sema), fHaveGrant(false) + explicit CSemaphoreGrant(CSemaphore& sema, bool fTry = false) noexcept : sem(&sema), fHaveGrant(false) { - if (fTry) + if (fTry) { TryAcquire(); - else + } else { Acquire(); + } } ~CSemaphoreGrant() @@ -391,7 +422,7 @@ public: Release(); } - operator bool() const + explicit operator bool() const noexcept { return fHaveGrant; } diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 787a196a0c..50f3f7d833 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -105,7 +105,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) // Send block connected notification, then stop the index without // sending a chainstate flushed notification. Prior to #24138, this // would cause the index to be corrupted and fail to reload. - ValidationInterfaceTest::BlockConnected(index, new_block, new_block_index); + ValidationInterfaceTest::BlockConnected(ChainstateRole::NORMAL, index, new_block, new_block_index); index.Stop(); } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index e46e085ee7..0dab2a2e97 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -61,7 +61,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) random_string = fuzzed_data_provider.ConsumeRandomLengthString(64); }, [&] { - connman.AddNode(random_string); + connman.AddNode({random_string, fuzzed_data_provider.ConsumeBool()}); }, [&] { connman.CheckIncomingNonce(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index e006653ca9..f3073c5c97 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -51,7 +51,7 @@ FUZZ_TARGET(golomb_rice) for (int i = 0; i < n; ++i) { elements.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider, 16)); } - CVectorWriter stream(SER_NETWORK, 0, golomb_rice_data, 0); + CVectorWriter stream{0, golomb_rice_data, 0}; WriteCompactSize(stream, static_cast<uint32_t>(elements.size())); BitStreamWriter<CVectorWriter> bitwriter(stream); if (!elements.empty()) { @@ -68,7 +68,7 @@ FUZZ_TARGET(golomb_rice) std::vector<uint64_t> decoded_deltas; { - SpanReader stream{SER_NETWORK, 0, golomb_rice_data}; + SpanReader stream{0, golomb_rice_data}; BitStreamReader<SpanReader> bitreader{stream}; const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); for (uint32_t i = 0; i < n; ++i) { @@ -80,7 +80,7 @@ FUZZ_TARGET(golomb_rice) { const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); - SpanReader stream{SER_NETWORK, 0, random_bytes}; + SpanReader stream{0, random_bytes}; uint32_t n; try { n = static_cast<uint32_t>(ReadCompactSize(stream)); diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 88d6e96eac..21d8dab536 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -328,6 +328,9 @@ void SimulationTest(Transport& initiator, Transport& responder, R& rng, FuzzedDa // Make sure all expected messages were received. assert(expected[0].empty()); assert(expected[1].empty()); + + // Compare session IDs. + assert(transports[0]->GetInfo().session_id == transports[1]->GetInfo().session_id); } std::unique_ptr<Transport> MakeV1Transport(NodeId nodeid) noexcept diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp new file mode 100644 index 0000000000..4c81c0b679 --- /dev/null +++ b/src/test/fuzz/package_eval.cpp @@ -0,0 +1,294 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <node/context.h> +#include <node/mempool_args.h> +#include <node/miner.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <test/util/mining.h> +#include <test/util/script.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <util/rbf.h> +#include <validation.h> +#include <validationinterface.h> + +using node::NodeContext; + +namespace { + +const TestingSetup* g_setup; +std::vector<COutPoint> g_outpoints_coinbase_init_mature; + +struct MockedTxPool : public CTxMemPool { + void RollingFeeUpdate() EXCLUSIVE_LOCKS_REQUIRED(!cs) + { + LOCK(cs); + lastRollingFeeUpdate = GetTime(); + blockSinceLastRollingFeeBump = true; + } +}; + +void initialize_tx_pool() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); + + for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; + if (i < COINBASE_MATURITY) { + // Remember the txids to avoid expensive disk access later on + g_outpoints_coinbase_init_mature.push_back(prevout); + } + } + SyncWithValidationInterfaceQueue(); +} + +struct OutpointsUpdater final : public CValidationInterface { + std::set<COutPoint>& m_mempool_outpoints; + + explicit OutpointsUpdater(std::set<COutPoint>& r) + : m_mempool_outpoints{r} {} + + void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t /* mempool_sequence */) override + { + // for coins spent we always want to be able to rbf so they're not removed + + // outputs from this tx can now be spent + for (uint32_t index{0}; index < tx->vout.size(); ++index) { + m_mempool_outpoints.insert(COutPoint{tx->GetHash(), index}); + } + } + + void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t /* mempool_sequence */) override + { + // outpoints spent by this tx are now available + for (const auto& input : tx->vin) { + // Could already exist if this was a replacement + m_mempool_outpoints.insert(input.prevout); + } + // outpoints created by this tx no longer exist + for (uint32_t index{0}; index < tx->vout.size(); ++index) { + m_mempool_outpoints.erase(COutPoint{tx->GetHash(), index}); + } + } +}; + +struct TransactionsDelta final : public CValidationInterface { + std::set<CTransactionRef>& m_added; + + explicit TransactionsDelta(std::set<CTransactionRef>& a) + : m_added{a} {} + + void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t /* mempool_sequence */) override + { + // Transactions may be entered and booted any number of times + m_added.insert(tx); + } + + void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t /* mempool_sequence */) override + { + // Transactions may be entered and booted any number of times + m_added.erase(tx); + } +}; + +void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chainstate) +{ + const auto time = ConsumeTime(fuzzed_data_provider, + chainstate.m_chain.Tip()->GetMedianTimePast() + 1, + std::numeric_limits<decltype(chainstate.m_chain.Tip()->nTime)>::max()); + SetMockTime(time); +} + +CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +{ + // Take the default options for tests... + CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; + + + // ...override specific options for this specific fuzz suite + mempool_opts.limits.ancestor_count = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 50); + mempool_opts.limits.ancestor_size_vbytes = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 202) * 1'000; + mempool_opts.limits.descendant_count = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 50); + mempool_opts.limits.descendant_size_vbytes = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 202) * 1'000; + mempool_opts.max_size_bytes = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 200) * 1'000'000; + mempool_opts.expiry = std::chrono::hours{fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 999)}; + nBytesPerSigOp = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(1, 999); + + mempool_opts.estimator = nullptr; + mempool_opts.check_ratio = 1; + mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); + + // ...and construct a CTxMemPool from it + return CTxMemPool{mempool_opts}; +} + +FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const auto& node = g_setup->m_node; + auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; + + MockTime(fuzzed_data_provider, chainstate); + + // All RBF-spendable outpoints outside of the unsubmitted package + std::set<COutPoint> mempool_outpoints; + std::map<COutPoint, CAmount> outpoints_value; + for (const auto& outpoint : g_outpoints_coinbase_init_mature) { + Assert(mempool_outpoints.insert(outpoint).second); + outpoints_value[outpoint] = 50 * COIN; + } + + auto outpoints_updater = std::make_shared<OutpointsUpdater>(mempool_outpoints); + RegisterSharedValidationInterface(outpoints_updater); + + CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + + chainstate.SetMempool(&tx_pool); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) + { + Assert(!mempool_outpoints.empty()); + + std::vector<CTransactionRef> txs; + + // Make packages of 1-to-26 transactions + const auto num_txs = (size_t) fuzzed_data_provider.ConsumeIntegralInRange<int>(1, 26); + std::set<COutPoint> package_outpoints; + while (txs.size() < num_txs) { + + // Last transaction in a package needs to be a child of parents to get further in validation + // so the last transaction to be generated(in a >1 package) must spend all package-made outputs + // Note that this test currently only spends package outputs in last transaction. + bool last_tx = num_txs > 1 && txs.size() == num_txs - 1; + + // Create transaction to add to the mempool + const CTransactionRef tx = [&] { + CMutableTransaction tx_mut; + tx_mut.nVersion = CTransaction::CURRENT_VERSION; + tx_mut.nLockTime = fuzzed_data_provider.ConsumeBool() ? 0 : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + // Last tx will sweep all outpoints in package + const auto num_in = last_tx ? package_outpoints.size() : fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size()); + const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<int>(1, mempool_outpoints.size() * 2); + + auto& outpoints = last_tx ? package_outpoints : mempool_outpoints; + + Assert(!outpoints.empty()); + + CAmount amount_in{0}; + for (size_t i = 0; i < num_in; ++i) { + // Pop random outpoint + auto pop = outpoints.begin(); + std::advance(pop, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, outpoints.size() - 1)); + const auto outpoint = *pop; + outpoints.erase(pop); + // no need to update or erase from outpoints_value + amount_in += outpoints_value.at(outpoint); + + // Create input + const auto sequence = ConsumeSequence(fuzzed_data_provider); + const auto script_sig = CScript{}; + const auto script_wit_stack = std::vector<std::vector<uint8_t>>{WITNESS_STACK_ELEM_OP_TRUE}; + CTxIn in; + in.prevout = outpoint; + in.nSequence = sequence; + in.scriptSig = script_sig; + in.scriptWitness.stack = script_wit_stack; + + tx_mut.vin.push_back(in); + } + const auto amount_fee = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, amount_in); + const auto amount_out = (amount_in - amount_fee) / num_out; + for (int i = 0; i < num_out; ++i) { + tx_mut.vout.emplace_back(amount_out, P2WSH_OP_TRUE); + } + // TODO vary transaction sizes to catch size-related issues + auto tx = MakeTransactionRef(tx_mut); + // Restore previously removed outpoints, except in-package outpoints + if (!last_tx) { + for (const auto& in : tx->vin) { + Assert(outpoints.insert(in.prevout).second); + } + // Cache the in-package outpoints being made + for (size_t i = 0; i < tx->vout.size(); ++i) { + package_outpoints.emplace(tx->GetHash(), i); + } + } + // We need newly-created values for the duration of this run + for (size_t i = 0; i < tx->vout.size(); ++i) { + outpoints_value[COutPoint(tx->GetHash(), i)] = tx->vout[i].nValue; + } + return tx; + }(); + txs.push_back(tx); + } + + if (fuzzed_data_provider.ConsumeBool()) { + MockTime(fuzzed_data_provider, chainstate); + } + if (fuzzed_data_provider.ConsumeBool()) { + tx_pool.RollingFeeUpdate(); + } + if (fuzzed_data_provider.ConsumeBool()) { + const auto& txid = fuzzed_data_provider.ConsumeBool() ? + txs.back()->GetHash() : + PickValue(fuzzed_data_provider, mempool_outpoints).hash; + const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); + tx_pool.PrioritiseTransaction(txid, delta); + } + + // Remember all added transactions + std::set<CTransactionRef> added; + auto txr = std::make_shared<TransactionsDelta>(added); + RegisterSharedValidationInterface(txr); + const bool bypass_limits = fuzzed_data_provider.ConsumeBool(); + + // When there are multiple transactions in the package, we call ProcessNewPackage(txs, test_accept=false) + // and AcceptToMemoryPool(txs.back(), test_accept=true). When there is only 1 transaction, we might flip it + // (the package is a test accept and ATMP is a submission). + auto single_submit = txs.size() == 1 && fuzzed_data_provider.ConsumeBool(); + + const auto result_package = WITH_LOCK(::cs_main, + return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit)); + // If something went wrong due to a package-specific policy, it might not return a + // validation result for the transaction. + if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { + auto it = result_package.m_tx_results.find(txs.back()->GetWitnessHash()); + Assert(it != result_package.m_tx_results.end()); + Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } + + const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, txs.back(), GetTime(), bypass_limits, /*test_accept=*/!single_submit)); + const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; + + SyncWithValidationInterfaceQueue(); + UnregisterSharedValidationInterface(txr); + + // There is only 1 transaction in the package. We did a test-package-accept and a ATMP + if (single_submit) { + Assert(accepted != added.empty()); + Assert(accepted == res.m_state.IsValid()); + if (accepted) { + Assert(added.size() == 1); + Assert(txs.back() == *added.begin()); + } + } else { + // This is empty if it fails early checks, or "full" if transactions are looked at deeper + Assert(result_package.m_tx_results.size() == txs.size() || result_package.m_tx_results.empty()); + } + } + + UnregisterSharedValidationInterface(outpoints_updater); + + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); +} +} // namespace diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 7e9a18e1d0..27bb60d6b6 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -80,6 +80,7 @@ const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ "gettxoutproof", // avoid prohibitively slow execution "importmempool", // avoid reading from disk "importwallet", // avoid reading from disk + "loadtxoutset", // avoid reading from disk "loadwallet", // avoid reading from disk "savemempool", // disabled as a precautionary measure: may take a file path argument in the future "setban", // avoid DNS lookups @@ -122,6 +123,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getblockstats", "getblocktemplate", "getchaintips", + "getchainstates", "getchaintxstats", "getconnectioncount", "getdeploymentinfo", diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 7862be2f21..66c862a6f9 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -54,7 +54,7 @@ CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; try { - SpanReader{SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; + SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; } catch (const std::ios_base::failure&) { throw std::runtime_error("Tx deserialization failure"); } @@ -68,7 +68,7 @@ std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; try { - SpanReader{SER_DISK, 0, CheckedParseHex(univalue[i].get_str())} >> txout; + SpanReader{0, CheckedParseHex(univalue[i].get_str())} >> txout; } catch (const std::ios_base::failure&) { throw std::runtime_error("Prevout invalid format"); } diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index a990797ca7..54afcef989 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,7 +122,7 @@ BOOST_AUTO_TEST_CASE(siphash) (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } - CHashWriter ss(SER_DISK, CLIENT_VERSION); + CHashWriter ss{CLIENT_VERSION}; CMutableTransaction tx; // Note these tests were originally written with tx.nVersion=1 // and the test would be affected by default tx version bumps if not fixed. diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 1e9c9d923b..5976aa3713 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1031,9 +1031,11 @@ class V2TransportTester bool m_test_initiator; //!< Whether m_transport is the initiator (true) or responder (false) std::vector<uint8_t> m_sent_garbage; //!< The garbage we've sent to m_transport. + std::vector<uint8_t> m_recv_garbage; //!< The garbage we've received from m_transport. std::vector<uint8_t> m_to_send; //!< Bytes we have queued up to send to m_transport. std::vector<uint8_t> m_received; //!< Bytes we have received from m_transport. std::deque<CSerializedNetMsg> m_msg_to_send; //!< Messages to be sent *by* m_transport to us. + bool m_sent_aad{false}; public: /** Construct a tester object. test_initiator: whether the tested transport is initiator. */ @@ -1131,8 +1133,7 @@ public: /** Schedule specified garbage to be sent to the transport. */ void SendGarbage(Span<const uint8_t> garbage) { - // Remember the specified garbage (so we can use it for constructing the garbage - // authentication packet). + // Remember the specified garbage (so we can use it as AAD). m_sent_garbage.assign(garbage.begin(), garbage.end()); // Schedule it for sending. Send(m_sent_garbage); @@ -1191,27 +1192,27 @@ public: Send(ciphertext); } - /** Schedule garbage terminator and authentication packet to be sent to the transport (only - * after ReceiveKey). */ - void SendGarbageTermAuth(size_t garb_auth_data_len = 0, bool garb_auth_ignore = false) + /** Schedule garbage terminator to be sent to the transport (only after ReceiveKey). */ + void SendGarbageTerm() { - // Generate random data to include in the garbage authentication packet (ignored by peer). - auto garb_auth_data = g_insecure_rand_ctx.randbytes<uint8_t>(garb_auth_data_len); // Schedule the garbage terminator to be sent. Send(m_cipher.GetSendGarbageTerminator()); - // Schedule the garbage authentication packet to be sent. - SendPacket(/*content=*/garb_auth_data, /*aad=*/m_sent_garbage, /*ignore=*/garb_auth_ignore); } /** Schedule version packet to be sent to the transport (only after ReceiveKey). */ void SendVersion(Span<const uint8_t> version_data = {}, bool vers_ignore = false) { - SendPacket(/*content=*/version_data, /*aad=*/{}, /*ignore=*/vers_ignore); + Span<const std::uint8_t> aad; + // Set AAD to garbage only for first packet. + if (!m_sent_aad) aad = m_sent_garbage; + SendPacket(/*content=*/version_data, /*aad=*/aad, /*ignore=*/vers_ignore); + m_sent_aad = true; } /** Expect a packet to have been received from transport, process it, and return its contents - * (only after ReceiveKey). By default, decoys are skipped. */ - std::vector<uint8_t> ReceivePacket(Span<const std::byte> aad = {}, bool skip_decoy = true) + * (only after ReceiveKey). Decoys are skipped. Optional associated authenticated data (AAD) is + * expected in the first received packet, no matter if that is a decoy or not. */ + std::vector<uint8_t> ReceivePacket(Span<const std::byte> aad = {}) { std::vector<uint8_t> contents; // Loop as long as there are ignored packets that are to be skipped. @@ -1232,16 +1233,18 @@ public: /*ignore=*/ignore, /*contents=*/MakeWritableByteSpan(contents)); BOOST_CHECK(ret); + // Don't expect AAD in further packets. + aad = {}; // Strip the processed packet's bytes off the front of the receive buffer. m_received.erase(m_received.begin(), m_received.begin() + size + BIP324Cipher::EXPANSION); - // Stop if the ignore bit is not set on this packet, or if we choose to not honor it. - if (!ignore || !skip_decoy) break; + // Stop if the ignore bit is not set on this packet. + if (!ignore) break; } return contents; } - /** Expect garbage, garbage terminator, and garbage auth packet to have been received, and - * process them (only after ReceiveKey). */ + /** Expect garbage and garbage terminator to have been received, and process them (only after + * ReceiveKey). */ void ReceiveGarbage() { // Figure out the garbage length. @@ -1252,18 +1255,15 @@ public: if (term_span == m_cipher.GetReceiveGarbageTerminator()) break; } // Copy the garbage to a buffer. - std::vector<uint8_t> garbage(m_received.begin(), m_received.begin() + garblen); + m_recv_garbage.assign(m_received.begin(), m_received.begin() + garblen); // Strip garbage + garbage terminator off the front of the receive buffer. m_received.erase(m_received.begin(), m_received.begin() + garblen + BIP324Cipher::GARBAGE_TERMINATOR_LEN); - // Process the expected garbage authentication packet. Such a packet still functions as one - // even when its ignore bit is set to true, so we do not skip decoy packets here. - ReceivePacket(/*aad=*/MakeByteSpan(garbage), /*skip_decoy=*/false); } /** Expect version packet to have been received, and process it (only after ReceiveKey). */ void ReceiveVersion() { - auto contents = ReceivePacket(); + auto contents = ReceivePacket(/*aad=*/MakeByteSpan(m_recv_garbage)); // Version packets from real BIP324 peers are expected to be empty, despite the fact that // this class supports *sending* non-empty version packets (to test that BIP324 peers // correctly ignore version packet contents). @@ -1321,6 +1321,14 @@ public: SendPacket(contents); } + /** Test whether the transport's session ID matches the session ID we expect. */ + void CompareSessionIDs() const + { + auto info = m_transport.GetInfo(); + BOOST_CHECK(info.session_id); + BOOST_CHECK(uint256(MakeUCharSpan(m_cipher.GetSessionID())) == *info.session_id); + } + /** Introduce a bit error in the data scheduled to be sent. */ void Damage() { @@ -1340,12 +1348,13 @@ BOOST_AUTO_TEST_CASE(v2transport_test) tester.SendKey(); tester.SendGarbage(); tester.ReceiveKey(); - tester.SendGarbageTermAuth(); + tester.SendGarbageTerm(); tester.SendVersion(); ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveGarbage(); tester.ReceiveVersion(); + tester.CompareSessionIDs(); auto msg_data_1 = g_insecure_rand_ctx.randbytes<uint8_t>(InsecureRandRange(100000)); auto msg_data_2 = g_insecure_rand_ctx.randbytes<uint8_t>(InsecureRandRange(1000)); tester.SendMessage(uint8_t(4), msg_data_1); // cmpctblock short id @@ -1380,12 +1389,13 @@ BOOST_AUTO_TEST_CASE(v2transport_test) auto ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveKey(); - tester.SendGarbageTermAuth(); + tester.SendGarbageTerm(); tester.SendVersion(); ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveGarbage(); tester.ReceiveVersion(); + tester.CompareSessionIDs(); auto msg_data_1 = g_insecure_rand_ctx.randbytes<uint8_t>(InsecureRandRange(100000)); auto msg_data_2 = g_insecure_rand_ctx.randbytes<uint8_t>(InsecureRandRange(1000)); tester.SendMessage(uint8_t(14), msg_data_1); // inv short id @@ -1408,10 +1418,6 @@ BOOST_AUTO_TEST_CASE(v2transport_test) bool initiator = InsecureRandBool(); /** Use either 0 bytes or the maximum possible (4095 bytes) garbage length. */ size_t garb_len = InsecureRandBool() ? 0 : V2Transport::MAX_GARBAGE_LEN; - /** Sometimes, use non-empty contents in the garbage authentication packet (which is to be ignored). */ - size_t garb_auth_data_len = InsecureRandBool() ? 0 : InsecureRandRange(100000); - /** Whether to set the ignore bit on the garbage authentication packet (it still functions as garbage authentication). */ - bool garb_ignore = InsecureRandBool(); /** How many decoy packets to send before the version packet. */ unsigned num_ignore_version = InsecureRandRange(10); /** What data to send in the version packet (ignored by BIP324 peers, but reserved for future extensions). */ @@ -1432,7 +1438,7 @@ BOOST_AUTO_TEST_CASE(v2transport_test) tester.SendGarbage(garb_len); } tester.ReceiveKey(); - tester.SendGarbageTermAuth(garb_auth_data_len, garb_ignore); + tester.SendGarbageTerm(); for (unsigned v = 0; v < num_ignore_version; ++v) { size_t ver_ign_data_len = InsecureRandBool() ? 0 : InsecureRandRange(1000); auto ver_ign_data = g_insecure_rand_ctx.randbytes<uint8_t>(ver_ign_data_len); @@ -1443,6 +1449,7 @@ BOOST_AUTO_TEST_CASE(v2transport_test) BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveGarbage(); tester.ReceiveVersion(); + tester.CompareSessionIDs(); for (unsigned d = 0; d < num_decoys_1; ++d) { auto decoy_data = g_insecure_rand_ctx.randbytes<uint8_t>(InsecureRandRange(1000)); tester.SendPacket(/*content=*/decoy_data, /*aad=*/{}, /*ignore=*/true); @@ -1476,7 +1483,7 @@ BOOST_AUTO_TEST_CASE(v2transport_test) tester.SendKey(); tester.SendGarbage(V2Transport::MAX_GARBAGE_LEN + 1); tester.ReceiveKey(); - tester.SendGarbageTermAuth(); + tester.SendGarbageTerm(); ret = tester.Interact(); BOOST_CHECK(!ret); } @@ -1489,7 +1496,7 @@ BOOST_AUTO_TEST_CASE(v2transport_test) auto ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveKey(); - tester.SendGarbageTermAuth(); + tester.SendGarbageTerm(); ret = tester.Interact(); BOOST_CHECK(!ret); } @@ -1514,12 +1521,13 @@ BOOST_AUTO_TEST_CASE(v2transport_test) // the first 15 of them match. garbage[len_before + 15] ^= (uint8_t(1) << InsecureRandRange(8)); tester.SendGarbage(garbage); - tester.SendGarbageTermAuth(); + tester.SendGarbageTerm(); tester.SendVersion(); ret = tester.Interact(); BOOST_REQUIRE(ret && ret->empty()); tester.ReceiveGarbage(); tester.ReceiveVersion(); + tester.CompareSessionIDs(); auto msg_data_1 = g_insecure_rand_ctx.randbytes<uint8_t>(4000000); // test that receiving 4M payload works auto msg_data_2 = g_insecure_rand_ctx.randbytes<uint8_t>(4000000); // test that sending 4M payload works tester.SendMessage(uint8_t(InsecureRandRange(223) + 33), {}); // unknown short id diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index d63bfb9603..94656b229e 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1470,7 +1470,7 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) static CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; - SpanReader{SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; + SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; return tx; } @@ -1480,7 +1480,7 @@ static std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) std::vector<CTxOut> prevouts; for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; - SpanReader{SER_DISK, 0, ParseHex(univalue[i].get_str())} >> txout; + SpanReader{0, ParseHex(univalue[i].get_str())} >> txout; prevouts.push_back(std::move(txout)); } return prevouts; @@ -1751,7 +1751,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) for (const auto& vec : vectors.getValues()) { auto txhex = ParseHex(vec["given"]["rawUnsignedTx"].get_str()); CMutableTransaction tx; - SpanReader{SER_NETWORK, PROTOCOL_VERSION, txhex} >> tx; + SpanReader{PROTOCOL_VERSION, txhex} >> tx; std::vector<CTxOut> utxos; for (const auto& utxo_spent : vec["given"]["utxosSpent"].getValues()) { auto script_bytes = ParseHex(utxo_spent["scriptPubKey"].get_str()); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 2f2bb6698c..d18d2623b1 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -176,7 +176,7 @@ BOOST_AUTO_TEST_CASE(vector_bool) std::vector<bool> vec2{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; BOOST_CHECK(vec1 == std::vector<uint8_t>(vec2.begin(), vec2.end())); - BOOST_CHECK(SerializeHash(vec1) == SerializeHash(vec2)); + BOOST_CHECK((HashWriter{} << vec1).GetHash() == (HashWriter{} << vec2).GetHash()); } BOOST_AUTO_TEST_CASE(noncanonical) diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index d1c0e1349e..178b16772b 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -78,7 +78,7 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un } // Serialize and hash - CHashWriter ss(SER_GETHASH, SERIALIZE_TRANSACTION_NO_WITNESS); + CHashWriter ss{SERIALIZE_TRANSACTION_NO_WITNESS}; ss << txTmp << nHashType; return ss.GetHash(); } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 8ff65b5377..f03f7c1da2 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -74,49 +74,49 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer) // point should yield the same results, even if the first test grew the // vector. - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 0, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 0, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); vch.clear(); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); vch.clear(); vch.resize(5, 0); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); vch.clear(); vch.resize(4, 0); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 3, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 3, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); vch.clear(); vch.resize(4, 0); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 4, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 4, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); vch.clear(); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); + CVectorWriter{INIT_PROTO_VERSION, vch, 0, bytes}; BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); + CVectorWriter{INIT_PROTO_VERSION, vch, 0, bytes}; BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); vch.clear(); vch.resize(4, 8); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, bytes, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); - CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); + CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, bytes, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); vch.clear(); } @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) { std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; - SpanReader reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; + SpanReader reader{INIT_PROTO_VERSION, vch}; BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); // Read a 4 bytes as a signed int from the beginning of the buffer. - SpanReader new_reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; + SpanReader new_reader{INIT_PROTO_VERSION, vch}; new_reader >> d; BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 BOOST_CHECK_EQUAL(new_reader.size(), 2U); @@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) { std::vector<uint8_t> data{0x82, 0xa7, 0x31}; - SpanReader reader{SER_NETWORK, INIT_PROTO_VERSION, data}; + SpanReader reader{INIT_PROTO_VERSION, data}; uint32_t varint = 0; // Deserialize into r-value reader >> VARINT(varint); diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 7f55916870..e2a88eacdd 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -109,7 +109,23 @@ CreateAndActivateUTXOSnapshot( 0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight())); } - return node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); + auto& new_active = node.chainman->ActiveChainstate(); + auto* tip = new_active.m_chain.Tip(); + + // Disconnect a block so that the snapshot chainstate will be ahead, otherwise + // it will refuse to activate. + // + // TODO this is a unittest-specific hack, and we should probably rethink how to + // better generate/activate snapshots in unittests. + if (tip->pprev) { + new_active.m_chain.SetTip(*(tip->pprev)); + } + + bool res = node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); + + // Restore the old tip. + new_active.m_chain.SetTip(*tip); + return res; } diff --git a/src/test/util/net.h b/src/test/util/net.h index 1684da777a..0d41cf550e 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -65,6 +65,7 @@ constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ NODE_WITNESS, NODE_COMPACT_FILTERS, NODE_NETWORK_LIMITED, + NODE_P2P_V2, }; constexpr NetPermissionFlags ALL_NET_PERMISSION_FLAGS[]{ diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp index 2d5562ae66..bcd6a7a7dc 100644 --- a/src/test/util/validation.cpp +++ b/src/test/util/validation.cpp @@ -22,7 +22,11 @@ void TestChainstateManager::JumpOutOfIbd() Assert(!IsInitialBlockDownload()); } -void ValidationInterfaceTest::BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) +void ValidationInterfaceTest::BlockConnected( + ChainstateRole role, + CValidationInterface& obj, + const std::shared_ptr<const CBlock>& block, + const CBlockIndex* pindex) { - obj.BlockConnected(block, pindex); + obj.BlockConnected(role, block, pindex); } diff --git a/src/test/util/validation.h b/src/test/util/validation.h index 64654f3fb6..45ef773409 100644 --- a/src/test/util/validation.h +++ b/src/test/util/validation.h @@ -19,7 +19,11 @@ struct TestChainstateManager : public ChainstateManager { class ValidationInterfaceTest { public: - static void BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex); + static void BlockConnected( + ChainstateRole role, + CValidationInterface& obj, + const std::shared_ptr<const CBlock>& block, + const CBlockIndex* pindex); }; #endif // BITCOIN_TEST_UTIL_VALIDATION_H diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index d1463634cc..411371f7c1 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -43,7 +43,7 @@ struct TestSubscriber final : public CValidationInterface { BOOST_CHECK_EQUAL(m_expected_tip, pindexNew->GetBlockHash()); } - void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override + void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override { BOOST_CHECK_EQUAL(m_expected_tip, block->hashPrevBlock); BOOST_CHECK_EQUAL(m_expected_tip, pindex->pprev->GetBlockHash()); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 7b7be4be9e..227d7d4633 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -30,30 +30,22 @@ using node::BlockManager; using node::KernelNotifications; using node::SnapshotMetadata; -BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) //! Basic tests for ChainstateManager. //! //! First create a legacy (IBD) chainstate, then create a snapshot chainstate. -BOOST_AUTO_TEST_CASE(chainstatemanager) +BOOST_FIXTURE_TEST_CASE(chainstatemanager, TestChain100Setup) { ChainstateManager& manager = *m_node.chainman; - CTxMemPool& mempool = *m_node.mempool; - std::vector<Chainstate*> chainstates; BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a legacy (IBD) chainstate. // - Chainstate& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); + Chainstate& c1 = manager.ActiveChainstate(); chainstates.push_back(&c1); - c1.InitCoinsDB( - /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); - WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); - c1.LoadGenesisBlock(); - BlockValidationState val_state; - BOOST_CHECK(c1.ActivateBestChain(val_state, nullptr)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); @@ -63,8 +55,9 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& active_chain = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); - BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 0); - + // Get to a valid assumeutxo tip (per chainparams); + mineBlocks(10); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 110); auto active_tip = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); @@ -74,19 +67,21 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // const uint256 snapshot_blockhash = active_tip->GetBlockHash(); - Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( - &mempool, snapshot_blockhash)); + Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot(snapshot_blockhash)); chainstates.push_back(&c2); - - BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); - c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); - WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); - c2.m_chain.SetTip(*active_tip); + { + LOCK(::cs_main); + c2.InitCoinsCache(1 << 23); + c2.CoinsTip().SetBestBlock(active_tip->GetBlockHash()); + c2.setBlockIndexCandidates.insert(manager.m_blockman.LookupBlockIndex(active_tip->GetBlockHash())); + c2.LoadChainTip(); + } BlockValidationState _; BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); + BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); @@ -97,13 +92,15 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& active_chain2 = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); - BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 0); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 110); + mineBlocks(1); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 111); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return c1.m_chain.Height()), 110); auto active_tip2 = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); - auto exp_tip2 = c2.m_chain.Tip(); - BOOST_CHECK_EQUAL(active_tip2, exp_tip2); - - BOOST_CHECK_EQUAL(exp_tip, exp_tip2); + BOOST_CHECK_EQUAL(active_tip, active_tip2->pprev); + BOOST_CHECK_EQUAL(active_tip, c1.m_chain.Tip()); + BOOST_CHECK_EQUAL(active_tip2, c2.m_chain.Tip()); // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); @@ -113,7 +110,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) { ChainstateManager& manager = *m_node.chainman; - CTxMemPool& mempool = *m_node.mempool; size_t max_cache = 10000; manager.m_total_coinsdb_cache = max_cache; @@ -125,9 +121,6 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) // Chainstate& c1 = manager.ActiveChainstate(); chainstates.push_back(&c1); - c1.InitCoinsDB( - /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); - { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); @@ -140,7 +133,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) // Create a snapshot-based chainstate. // CBlockIndex* snapshot_base{WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()[manager.ActiveChain().Height() / 2])}; - Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, *snapshot_base->phashBlock)); + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(*snapshot_base->phashBlock)); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -289,10 +282,10 @@ struct SnapshotTestSetup : TestChain100Setup { BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); } - const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); + const auto& au_data = ::Params().AssumeutxoForHeight(snapshot_height); const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); - BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); + BOOST_CHECK_EQUAL(tip->nChainTx, au_data->nChainTx); // To be checked against later when we try loading a subsequent snapshot. uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; @@ -426,18 +419,24 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) { ChainstateManager& chainman = *Assert(m_node.chainman); - CTxMemPool& mempool = *m_node.mempool; Chainstate& cs1 = chainman.ActiveChainstate(); int num_indexes{0}; int num_assumed_valid{0}; + // Blocks in range [assumed_valid_start_idx, last_assumed_valid_idx) will be + // marked as assumed-valid and not having data. const int expected_assumed_valid{20}; - const int last_assumed_valid_idx{40}; + const int last_assumed_valid_idx{111}; const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; + // Mine to height 120, past the hardcoded regtest assumeutxo snapshot at + // height 110 + mineBlocks(20); + CBlockIndex* validated_tip{nullptr}; CBlockIndex* assumed_base{nullptr}; CBlockIndex* assumed_tip{WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip())}; + BOOST_CHECK_EQUAL(assumed_tip->nHeight, 120); auto reload_all_block_indexes = [&]() { // For completeness, we also reset the block sequence counters to @@ -463,7 +462,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) LOCK(::cs_main); auto index = cs1.m_chain[i]; - // Blocks with heights in range [20, 40) are marked ASSUMED_VALID + // Blocks with heights in range [91, 110] are marked ASSUMED_VALID if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; } @@ -489,7 +488,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) // Note: cs2's tip is not set when ActivateExistingSnapshot is called. Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.ActivateExistingSnapshot(&mempool, *assumed_base->phashBlock)); + return chainman.ActivateExistingSnapshot(*assumed_base->phashBlock)); // Set tip of the fully validated chain to be the validated tip cs1.m_chain.SetTip(*validated_tip); @@ -497,10 +496,36 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) // Set tip of the assume-valid-based chain to the assume-valid block cs2.m_chain.SetTip(*assumed_base); + // Sanity check test variables. + BOOST_CHECK_EQUAL(num_indexes, 121); // 121 total blocks, including genesis + BOOST_CHECK_EQUAL(assumed_tip->nHeight, 120); // original chain has height 120 + BOOST_CHECK_EQUAL(validated_tip->nHeight, 90); // current cs1 chain has height 90 + BOOST_CHECK_EQUAL(assumed_base->nHeight, 110); // current cs2 chain has height 110 + + // Regenerate cs1.setBlockIndexCandidates and cs2.setBlockIndexCandidate and + // check contents below. reload_all_block_indexes(); - // The fully validated chain should have the current validated tip - // and the assumed valid base as candidates. + // The fully validated chain should only have the current validated tip and + // the assumed valid base as candidates, blocks 90 and 110. Specifically: + // + // - It does not have blocks 0-89 because they contain less work than the + // chain tip. + // + // - It has block 90 because it has data and equal work to the chain tip, + // (since it is the chain tip). + // + // - It does not have blocks 91-109 because they do not contain data. + // + // - It has block 110 even though it does not have data, because + // LoadBlockIndex has a special case to always add the snapshot block as a + // candidate. The special case is only actually intended to apply to the + // snapshot chainstate cs2, not the background chainstate cs1, but it is + // written broadly and applies to both. + // + // - It does not have any blocks after height 110 because cs1 is a background + // chainstate, and only blocks where are ancestors of the snapshot block + // are added as candidates for the background chainstate. BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 2); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_base), 1); @@ -508,8 +533,25 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) // The assumed-valid tolerant chain has the assumed valid base as a // candidate, but otherwise has none of the assumed-valid (which do not // HAVE_DATA) blocks as candidates. + // + // Specifically: + // - All blocks below height 110 are not candidates, because cs2 chain tip + // has height 110 and they have less work than it does. + // + // - Block 110 is a candidate even though it does not have data, because it + // is the snapshot block, which is assumed valid. + // + // - Blocks 111-120 are added because they have data. + + // Check that block 90 is absent BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 0); + // Check that block 109 is absent + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_base->pprev), 0); + // Check that block 110 is present + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_base), 1); + // Check that block 120 is present BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); + // Check that 11 blocks total are present. BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes - last_assumed_valid_idx + 1); } diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index d00f2ff4d1..d34d98c219 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -132,17 +132,17 @@ BOOST_AUTO_TEST_CASE(test_assumeutxo) std::vector<int> bad_heights{0, 100, 111, 115, 209, 211}; for (auto empty : bad_heights) { - const auto out = ExpectedAssumeutxo(empty, *params); + const auto out = params->AssumeutxoForHeight(empty); BOOST_CHECK(!out); } - const auto out110 = *ExpectedAssumeutxo(110, *params); + const auto out110 = *params->AssumeutxoForHeight(110); BOOST_CHECK_EQUAL(out110.hash_serialized.ToString(), "1ebbf5850204c0bdb15bf030f47c7fe91d45c44c712697e4509ba67adb01c618"); BOOST_CHECK_EQUAL(out110.nChainTx, 110U); - const auto out210 = *ExpectedAssumeutxo(200, *params); - BOOST_CHECK_EQUAL(out210.hash_serialized.ToString(), "51c8d11d8b5c1de51543c579736e786aa2736206d1e11e627568029ce092cf62"); - BOOST_CHECK_EQUAL(out210.nChainTx, 200U); + const auto out110_2 = *params->AssumeutxoForBlockhash(uint256S("0x696e92821f65549c7ee134edceeeeaaa4105647a3c4fd9f298c0aec0ab50425c")); + BOOST_CHECK_EQUAL(out110_2.hash_serialized.ToString(), "1ebbf5850204c0bdb15bf030f47c7fe91d45c44c712697e4509ba67adb01c618"); + BOOST_CHECK_EQUAL(out110_2.nChainTx, 110U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp index fcd0b25b38..5979441057 100644 --- a/src/test/validationinterface_tests.cpp +++ b/src/test/validationinterface_tests.cpp @@ -8,6 +8,7 @@ #include <scheduler.h> #include <test/util/setup_common.h> #include <util/check.h> +#include <kernel/chain.h> #include <validationinterface.h> #include <atomic> diff --git a/src/util/vector.h b/src/util/vector.h index 40ff73c293..1513562f1b 100644 --- a/src/util/vector.h +++ b/src/util/vector.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_UTIL_VECTOR_H #define BITCOIN_UTIL_VECTOR_H +#include <functional> #include <initializer_list> +#include <optional> #include <type_traits> #include <utility> #include <vector> @@ -67,4 +69,15 @@ inline void ClearShrink(V& v) noexcept V{}.swap(v); } +template<typename V, typename L> +inline std::optional<V> FindFirst(const std::vector<V>& vec, const L fnc) +{ + for (const auto& el : vec) { + if (fnc(el)) { + return el; + } + } + return std::nullopt; +} + #endif // BITCOIN_UTIL_VECTOR_H diff --git a/src/validation.cpp b/src/validation.cpp index 4acd5c7cb0..30b3dde74f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,6 +5,7 @@ #include <validation.h> +#include <kernel/chain.h> #include <kernel/coinstats.h> #include <kernel/mempool_persist.h> @@ -68,6 +69,7 @@ #include <optional> #include <string> #include <utility> +#include <tuple> using kernel::CCoinsStats; using kernel::CoinStatsHashType; @@ -2551,11 +2553,14 @@ bool Chainstate::FlushStateToDisk( if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); - m_blockman.FindFilesToPruneManual(setFilesToPrune, std::min(last_prune, nManualPruneHeight), m_chain.Height()); + m_blockman.FindFilesToPruneManual( + setFilesToPrune, + std::min(last_prune, nManualPruneHeight), + *this, m_chainman); } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, m_chainman.GetParams().PruneAfterHeight(), m_chain.Height(), last_prune, m_chainman.IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, last_prune, *this, m_chainman); m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { @@ -2594,7 +2599,11 @@ bool Chainstate::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); // First make sure all block and undo data is flushed to disk. - m_blockman.FlushBlockFile(); + // TODO: Handle return error, or add detailed comment why it is + // safe to not return an error upon failure. + if (!m_blockman.FlushChainstateBlockFile(m_chain.Height())) { + LogPrintLevel(BCLog::VALIDATION, BCLog::Level::Warning, "%s: Failed to flush block file.\n", __func__); + } } // Then update all block file information (which may refer to block and undo files). @@ -2641,7 +2650,7 @@ bool Chainstate::FlushStateToDisk( } if (full_flush_completed) { // Update best block in wallet (so we can detect restored wallets). - GetMainSignals().ChainStateFlushed(m_chain.GetLocator()); + GetMainSignals().ChainStateFlushed(this->GetRole(), m_chain.GetLocator()); } } catch (const std::runtime_error& e) { return FatalError(m_chainman.GetNotifications(), state, std::string("System error while flushing: ") + e.what()); @@ -3188,6 +3197,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; + bool exited_ibd{false}; do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -3201,6 +3211,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< LOCK(cs_main); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed LOCK(MempoolMutex()); + const bool was_in_ibd = m_chainman.IsInitialBlockDownload(); CBlockIndex* starting_tip = m_chain.Tip(); bool blocks_connected = false; do { @@ -3233,7 +3244,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); - GetMainSignals().BlockConnected(trace.pblock, trace.pindex); + GetMainSignals().BlockConnected(this->GetRole(), trace.pblock, trace.pindex); } // This will have been toggled in @@ -3248,16 +3259,21 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< if (!blocks_connected) return true; const CBlockIndex* pindexFork = m_chain.FindFork(starting_tip); - bool fInitialDownload = m_chainman.IsInitialBlockDownload(); + bool still_in_ibd = m_chainman.IsInitialBlockDownload(); + + if (was_in_ibd && !still_in_ibd) { + // Active chainstate has exited IBD. + exited_ibd = true; + } // Notify external listeners about the new tip. // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected - if (pindexFork != pindexNewTip) { + if (this == &m_chainman.ActiveChainstate() && pindexFork != pindexNewTip) { // Notify ValidationInterface subscribers - GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); + GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, still_in_ibd); // Always notify the UI if a new block tip was connected - if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(fInitialDownload), *pindexNewTip))) { + if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to // the caller so the caller could distinguish between @@ -3268,8 +3284,25 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< } // When we reach this point, we switched to a new tip (stored in pindexNewTip). + if (exited_ibd) { + // If a background chainstate is in use, we may need to rebalance our + // allocation of caches once a chainstate exits initial block download. + LOCK(::cs_main); + m_chainman.MaybeRebalanceCaches(); + } + if (WITH_LOCK(::cs_main, return m_disabled)) { // Background chainstate has reached the snapshot base block, so exit. + + // Restart indexes to resume indexing for all blocks unique to the snapshot + // chain. This resumes indexing "in order" from where the indexing on the + // background validation chain left off. + // + // This cannot be done while holding cs_main (within + // MaybeCompleteSnapshotValidation) or a cs_main deadlock will occur. + if (m_chainman.restart_indexes) { + m_chainman.restart_indexes(); + } break; } @@ -3506,7 +3539,8 @@ void Chainstate::ResetBlockFailureFlags(CBlockIndex *pindex) { void Chainstate::TryAddBlockIndexCandidate(CBlockIndex* pindex) { AssertLockHeld(cs_main); - // The block only is a candidate for the most-work-chain if it has more work than our current tip. + // The block only is a candidate for the most-work-chain if it has the same + // or more work than our current tip. if (m_chain.Tip() != nullptr && setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { return; } @@ -4144,6 +4178,12 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); } + Chainstate* bg_chain{WITH_LOCK(cs_main, return BackgroundSyncInProgress() ? m_ibd_chainstate.get() : nullptr)}; + BlockValidationState bg_state; + if (bg_chain && !bg_chain->ActivateBestChain(bg_state, block)) { + return error("%s: [background] ActivateBestChain failed (%s)", __func__, bg_state.ToString()); + } + return true; } @@ -4271,7 +4311,7 @@ VerifyDBResult CVerifyDB::VerifyDB( bool skipped_l3_checks{false}; LogPrintf("Verification progress: 0%%\n"); - const bool is_snapshot_cs{!chainstate.m_from_snapshot_blockhash}; + const bool is_snapshot_cs{chainstate.m_from_snapshot_blockhash}; for (pindex = chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { const int percentageDone = std::max(1, std::min(99, (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); @@ -4502,7 +4542,7 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret{m_blockman.LoadBlockIndexDB()}; + bool ret{m_blockman.LoadBlockIndexDB(SnapshotBlockhash())}; if (!ret) return false; m_blockman.ScanAndUnlinkAlreadyPrunedFiles(); @@ -4798,6 +4838,10 @@ void ChainstateManager::CheckBlockIndex() CBlockIndex* pindexFirstAssumeValid = nullptr; // Oldest ancestor of pindex which has BLOCK_ASSUMED_VALID while (pindex != nullptr) { nNodes++; + if (pindex->pprev && pindex->nTx > 0) { + // nChainTx should increase monotonically + assert(pindex->pprev->nChainTx <= pindex->nChainTx); + } if (pindexFirstAssumeValid == nullptr && pindex->nStatus & BLOCK_ASSUMED_VALID) pindexFirstAssumeValid = pindex; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) { @@ -5089,19 +5133,7 @@ Chainstate& ChainstateManager::InitializeChainstate(CTxMemPool* mempool) return *m_active_chainstate; } -const AssumeutxoData* ExpectedAssumeutxo( - const int height, const CChainParams& chainparams) -{ - const MapAssumeutxo& valid_assumeutxos_map = chainparams.Assumeutxo(); - const auto assumeutxo_found = valid_assumeutxos_map.find(height); - - if (assumeutxo_found != valid_assumeutxos_map.end()) { - return &assumeutxo_found->second; - } - return nullptr; -} - -static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) +[[nodiscard]] static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); @@ -5153,6 +5185,14 @@ bool ChainstateManager::ActivateSnapshot( return false; } + { + LOCK(::cs_main); + if (Assert(m_active_chainstate->GetMempool())->size() > 0) { + LogPrintf("[snapshot] can't activate a snapshot when mempool not empty\n"); + return false; + } + } + int64_t current_coinsdb_cache_size{0}; int64_t current_coinstip_cache_size{0}; @@ -5198,19 +5238,8 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); } - bool snapshot_ok = this->PopulateAndValidateSnapshot( - *snapshot_chainstate, coins_file, metadata); - - // If not in-memory, persist the base blockhash for use during subsequent - // initialization. - if (!in_memory) { - LOCK(::cs_main); - if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) { - snapshot_ok = false; - } - } - if (!snapshot_ok) { - LOCK(::cs_main); + auto cleanup_bad_snapshot = [&](const char* reason) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + LogPrintf("[snapshot] activation failed - %s\n", reason); this->MaybeRebalanceCaches(); // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir @@ -5227,23 +5256,48 @@ bool ChainstateManager::ActivateSnapshot( } } return false; - } + }; - { + if (!this->PopulateAndValidateSnapshot(*snapshot_chainstate, coins_file, metadata)) { LOCK(::cs_main); - assert(!m_snapshot_chainstate); - m_snapshot_chainstate.swap(snapshot_chainstate); - const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(); - assert(chaintip_loaded); - - m_active_chainstate = m_snapshot_chainstate.get(); + return cleanup_bad_snapshot("population failed"); + } - LogPrintf("[snapshot] successfully activated snapshot %s\n", base_blockhash.ToString()); - LogPrintf("[snapshot] (%.2f MB)\n", - m_snapshot_chainstate->CoinsTip().DynamicMemoryUsage() / (1000 * 1000)); + LOCK(::cs_main); // cs_main required for rest of snapshot activation. - this->MaybeRebalanceCaches(); + // Do a final check to ensure that the snapshot chainstate is actually a more + // work chain than the active chainstate; a user could have loaded a snapshot + // very late in the IBD process, and we wouldn't want to load a useless chainstate. + if (!CBlockIndexWorkComparator()(ActiveTip(), snapshot_chainstate->m_chain.Tip())) { + return cleanup_bad_snapshot("work does not exceed active chainstate"); + } + // If not in-memory, persist the base blockhash for use during subsequent + // initialization. + if (!in_memory) { + if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) { + return cleanup_bad_snapshot("could not write base blockhash"); + } } + + assert(!m_snapshot_chainstate); + m_snapshot_chainstate.swap(snapshot_chainstate); + const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(); + assert(chaintip_loaded); + + // Transfer possession of the mempool to the snapshot chainstate. + // Mempool is empty at this point because we're still in IBD. + Assert(m_active_chainstate->m_mempool->size() == 0); + Assert(!m_snapshot_chainstate->m_mempool); + m_snapshot_chainstate->m_mempool = m_active_chainstate->m_mempool; + m_active_chainstate->m_mempool = nullptr; + m_active_chainstate = m_snapshot_chainstate.get(); + m_blockman.m_snapshot_height = this->GetSnapshotBaseHeight(); + + LogPrintf("[snapshot] successfully activated snapshot %s\n", base_blockhash.ToString()); + LogPrintf("[snapshot] (%.2f MB)\n", + m_snapshot_chainstate->CoinsTip().DynamicMemoryUsage() / (1000 * 1000)); + + this->MaybeRebalanceCaches(); return true; } @@ -5285,7 +5339,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( CBlockIndex* snapshot_start_block = WITH_LOCK(::cs_main, return m_blockman.LookupBlockIndex(base_blockhash)); if (!snapshot_start_block) { - // Needed for ComputeUTXOStats and ExpectedAssumeutxo to determine the + // Needed for ComputeUTXOStats to determine the // height and to avoid a crash when base_blockhash.IsNull() LogPrintf("[snapshot] Did not find snapshot start blockheader %s\n", base_blockhash.ToString()); @@ -5293,7 +5347,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } int base_height = snapshot_start_block->nHeight; - auto maybe_au_data = ExpectedAssumeutxo(base_height, GetParams()); + const auto& maybe_au_data = GetParams().AssumeutxoForHeight(base_height); if (!maybe_au_data) { LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " @@ -5303,6 +5357,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( const AssumeutxoData& au_data = *maybe_au_data; + // This work comparison is a duplicate check with the one performed later in + // ActivateSnapshot(), but is done so that we avoid doing the long work of staging + // a snapshot that isn't actually usable. + if (WITH_LOCK(::cs_main, return !CBlockIndexWorkComparator()(ActiveTip(), snapshot_start_block))) { + LogPrintf("[snapshot] activation failed - height does not exceed active chainstate\n"); + return false; + } + COutPoint outpoint; Coin coin; const uint64_t coins_count = metadata.m_coins_count; @@ -5562,7 +5624,7 @@ SnapshotCompletionResult ChainstateManager::MaybeCompleteSnapshotValidation() CCoinsViewDB& ibd_coins_db = m_ibd_chainstate->CoinsDB(); m_ibd_chainstate->ForceFlushStateToDisk(); - auto maybe_au_data = ExpectedAssumeutxo(curr_height, m_options.chainparams); + const auto& maybe_au_data = m_options.chainparams.AssumeutxoForHeight(curr_height); if (!maybe_au_data) { LogPrintf("[snapshot] assumeutxo data not found for height " "(%d) - refusing to validate snapshot\n", curr_height); @@ -5714,16 +5776,22 @@ bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) LogPrintf("[snapshot] detected active snapshot chainstate (%s) - loading\n", fs::PathToString(*path)); - this->ActivateExistingSnapshot(mempool, *base_blockhash); + this->ActivateExistingSnapshot(*base_blockhash); return true; } -Chainstate& ChainstateManager::ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) +Chainstate& ChainstateManager::ActivateExistingSnapshot(uint256 base_blockhash) { assert(!m_snapshot_chainstate); m_snapshot_chainstate = - std::make_unique<Chainstate>(mempool, m_blockman, *this, base_blockhash); + std::make_unique<Chainstate>(nullptr, m_blockman, *this, base_blockhash); LogPrintf("[snapshot] switching active chainstate to %s\n", m_snapshot_chainstate->ToString()); + + // Mempool is empty at this point because we're still in IBD. + Assert(m_active_chainstate->m_mempool->size() == 0); + Assert(!m_snapshot_chainstate->m_mempool); + m_snapshot_chainstate->m_mempool = m_active_chainstate->m_mempool; + m_active_chainstate->m_mempool = nullptr; m_active_chainstate = m_snapshot_chainstate.get(); return *m_snapshot_chainstate; } @@ -5740,15 +5808,20 @@ bool IsBIP30Unspendable(const CBlockIndex& block_index) (block_index.nHeight==91812 && block_index.GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f")); } -util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() +static fs::path GetSnapshotCoinsDBPath(Chainstate& cs) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); // Should never be called on a non-snapshot chainstate. - assert(m_from_snapshot_blockhash); - auto storage_path_maybe = this->CoinsDB().StoragePath(); + assert(cs.m_from_snapshot_blockhash); + auto storage_path_maybe = cs.CoinsDB().StoragePath(); // Should never be called with a non-existent storage path. assert(storage_path_maybe); - fs::path snapshot_datadir = *storage_path_maybe; + return *storage_path_maybe; +} + +util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() +{ + fs::path snapshot_datadir = GetSnapshotCoinsDBPath(*this); // Coins views no longer usable. m_coins_views.reset(); @@ -5779,6 +5852,33 @@ util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() return {}; } +bool ChainstateManager::DeleteSnapshotChainstate() +{ + AssertLockHeld(::cs_main); + Assert(m_snapshot_chainstate); + Assert(m_ibd_chainstate); + + fs::path snapshot_datadir = GetSnapshotCoinsDBPath(*m_snapshot_chainstate); + if (!DeleteCoinsDBFromDisk(snapshot_datadir, /*is_snapshot=*/ true)) { + LogPrintf("Deletion of %s failed. Please remove it manually to continue reindexing.\n", + fs::PathToString(snapshot_datadir)); + return false; + } + m_active_chainstate = m_ibd_chainstate.get(); + m_snapshot_chainstate.reset(); + return true; +} + +ChainstateRole Chainstate::GetRole() const +{ + if (m_chainman.GetAll().size() <= 1) { + return ChainstateRole::NORMAL; + } + return (this != &m_chainman.ActiveChainstate()) ? + ChainstateRole::BACKGROUND : + ChainstateRole::ASSUMEDVALID; +} + const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const { return m_active_chainstate ? m_active_chainstate->SnapshotBase() : nullptr; @@ -5876,3 +5976,38 @@ bool ChainstateManager::ValidatedSnapshotCleanup() } return true; } + +Chainstate& ChainstateManager::GetChainstateForIndexing() +{ + // We can't always return `m_ibd_chainstate` because after background validation + // has completed, `m_snapshot_chainstate == m_active_chainstate`, but it can be + // indexed. + return (this->GetAll().size() > 1) ? *m_ibd_chainstate : *m_active_chainstate; +} + +std::pair<int, int> ChainstateManager::GetPruneRange(const Chainstate& chainstate, int last_height_can_prune) +{ + if (chainstate.m_chain.Height() <= 0) { + return {0, 0}; + } + int prune_start{0}; + + if (this->GetAll().size() > 1 && m_snapshot_chainstate.get() == &chainstate) { + // Leave the blocks in the background IBD chain alone if we're pruning + // the snapshot chain. + prune_start = *Assert(GetSnapshotBaseHeight()) + 1; + } + + int max_prune = std::max<int>( + 0, chainstate.m_chain.Height() - static_cast<int>(MIN_BLOCKS_TO_KEEP)); + + // last block to prune is the lesser of (caller-specified height, MIN_BLOCKS_TO_KEEP from the tip) + // + // While you might be tempted to prune the background chainstate more + // aggressively (i.e. fewer MIN_BLOCKS_TO_KEEP), this won't work with index + // building - specifically blockfilterindex requires undo data, and if + // we don't maintain this trailing window, we hit indexing failures. + int prune_end = std::min(last_height_can_prune, max_prune); + + return {prune_start, prune_end}; +} diff --git a/src/validation.h b/src/validation.h index 3f0a2312b5..94a00e44a4 100644 --- a/src/validation.h +++ b/src/validation.h @@ -13,6 +13,7 @@ #include <arith_uint256.h> #include <attributes.h> #include <chain.h> +#include <kernel/chain.h> #include <consensus/amount.h> #include <deploymentstatus.h> #include <kernel/chainparams.h> @@ -511,6 +512,12 @@ public: ChainstateManager& chainman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); + //! Return the current role of the chainstate. See `ChainstateManager` + //! documentation for a description of the different types of chainstates. + //! + //! @sa ChainstateRole + ChainstateRole GetRole() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory * cache is initialized separately. @@ -848,9 +855,6 @@ private: //! Points to either the ibd or snapshot chainstate; indicates our //! most-work chain. //! - //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). - //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. @@ -881,13 +885,6 @@ private: /** Most recent headers presync progress update, for rate-limiting. */ std::chrono::time_point<std::chrono::steady_clock> m_last_presync_update GUARDED_BY(::cs_main) {}; - //! Returns nullptr if no snapshot has been loaded. - const CBlockIndex* GetSnapshotBaseBlock() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - - //! Return the height of the base block of the snapshot in use, if one exists, else - //! nullopt. - std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> m_warningcache GUARDED_BY(::cs_main); //! Return true if a chainstate is considered usable. @@ -904,6 +901,10 @@ public: explicit ChainstateManager(const util::SignalInterrupt& interrupt, Options options, node::BlockManager::Options blockman_options); + //! Function to restart active indexes; set dynamically to avoid a circular + //! dependency on `base/index.cpp`. + std::function<void()> restart_indexes = std::function<void()>(); + const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } @@ -1034,12 +1035,25 @@ public: //! Otherwise, revert to using the ibd chainstate and shutdown. SnapshotCompletionResult MaybeCompleteSnapshotValidation() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! Returns nullptr if no snapshot has been loaded. + const CBlockIndex* GetSnapshotBaseBlock() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! The most-work chain. Chainstate& ActiveChainstate() const; CChain& ActiveChain() const EXCLUSIVE_LOCKS_REQUIRED(GetMutex()) { return ActiveChainstate().m_chain; } int ActiveHeight() const EXCLUSIVE_LOCKS_REQUIRED(GetMutex()) { return ActiveChain().Height(); } CBlockIndex* ActiveTip() const EXCLUSIVE_LOCKS_REQUIRED(GetMutex()) { return ActiveChain().Tip(); } + //! The state of a background sync (for net processing) + bool BackgroundSyncInProgress() const EXCLUSIVE_LOCKS_REQUIRED(GetMutex()) { + return IsUsable(m_snapshot_chainstate.get()) && IsUsable(m_ibd_chainstate.get()); + } + + //! The tip of the background sync chain + const CBlockIndex* GetBackgroundSyncTip() const EXCLUSIVE_LOCKS_REQUIRED(GetMutex()) { + return BackgroundSyncInProgress() ? m_ibd_chainstate->m_chain.Tip() : nullptr; + } + node::BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); @@ -1193,10 +1207,13 @@ public: void ResetChainstates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! Remove the snapshot-based chainstate and all on-disk artifacts. + //! Used when reindex{-chainstate} is called during snapshot use. + [[nodiscard]] bool DeleteSnapshotChainstate() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! Switch the active chainstate to one based on a UTXO snapshot that was loaded //! previously. - Chainstate& ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + Chainstate& ActivateExistingSnapshot(uint256 base_blockhash) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! If we have validated a snapshot chain during this runtime, copy its //! chainstate directory over to the main `chainstate` location, completing @@ -1209,6 +1226,26 @@ public: //! @sa node/chainstate:LoadChainstate() bool ValidatedSnapshotCleanup() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + //! @returns the chainstate that indexes should consult when ensuring that an + //! index is synced with a chain where we can expect block index entries to have + //! BLOCK_HAVE_DATA beneath the tip. + //! + //! In other words, give us the chainstate for which we can reasonably expect + //! that all blocks beneath the tip have been indexed. In practice this means + //! when using an assumed-valid chainstate based upon a snapshot, return only the + //! fully validated chain. + Chainstate& GetChainstateForIndexing() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Return the [start, end] (inclusive) of block heights we can prune. + //! + //! start > end is possible, meaning no blocks can be pruned. + std::pair<int, int> GetPruneRange( + const Chainstate& chainstate, int last_height_can_prune) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Return the height of the base block of the snapshot in use, if one exists, else + //! nullopt. + std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + ~ChainstateManager(); }; @@ -1231,15 +1268,6 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) return DeploymentEnabled(chainman.GetConsensus(), dep); } -/** - * Return the expected assumeutxo value for a given height, if one exists. - * - * @param[in] height Get the assumeutxo value for this height. - * - * @returns empty if no assumeutxo configuration exists for the given height. - */ -const AssumeutxoData* ExpectedAssumeutxo(const int height, const CChainParams& params); - /** Identifies blocks that overwrote an existing coinbase output in the UTXO set (see BIP30) */ bool IsBIP30Repeat(const CBlockIndex& block_index); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index d344c8bfbd..9241395ad5 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -8,6 +8,7 @@ #include <attributes.h> #include <chain.h> #include <consensus/validation.h> +#include <kernel/chain.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -223,9 +224,9 @@ void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef& tx, MemP RemovalReasonToString(reason)); } -void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { - auto event = [pblock, pindex, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.BlockConnected(pblock, pindex); }); +void CMainSignals::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { + auto event = [role, pblock, pindex, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.BlockConnected(role, pblock, pindex); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, pblock->GetHash().ToString(), @@ -242,9 +243,9 @@ void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock pindex->nHeight); } -void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { - auto event = [locator, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.ChainStateFlushed(locator); }); +void CMainSignals::ChainStateFlushed(ChainstateRole role, const CBlockLocator &locator) { + auto event = [role, locator, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.ChainStateFlushed(role, locator); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s", __func__, locator.IsNull() ? "null" : locator.vHave.front().ToString()); diff --git a/src/validationinterface.h b/src/validationinterface.h index 8c20cc8ffb..eb15aa4d5f 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -7,6 +7,7 @@ #define BITCOIN_VALIDATIONINTERFACE_H #include <kernel/cs_main.h> +#include <kernel/chain.h> #include <primitives/transaction.h> // CTransaction(Ref) #include <sync.h> @@ -87,7 +88,7 @@ protected: * but may not be called on every intermediate tip. If the latter behavior is desired, * subscribe to BlockConnected() instead. * - * Called on a background thread. + * Called on a background thread. Only called for the active chainstate. */ virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} /** @@ -136,11 +137,12 @@ protected: * * Called on a background thread. */ - virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex) {} + virtual void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex) {} /** * Notifies listeners of a block being disconnected * - * Called on a background thread. + * Called on a background thread. Only called for the active chainstate, since + * background chainstates should never disconnect blocks. */ virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) {} /** @@ -159,17 +161,18 @@ protected: * * Called on a background thread. */ - virtual void ChainStateFlushed(const CBlockLocator &locator) {} + virtual void ChainStateFlushed(ChainstateRole role, const CBlockLocator &locator) {} /** * Notifies listeners of a block validation result. * If the provided BlockValidationState IsValid, the provided block * is guaranteed to be the current best block at the time the - * callback was generated (not necessarily now) + * callback was generated (not necessarily now). */ virtual void BlockChecked(const CBlock&, const BlockValidationState&) {} /** * Notifies listeners that a block which builds directly on our current tip - * has been received and connected to the headers tree, though not validated yet */ + * has been received and connected to the headers tree, though not validated yet. + */ virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; friend class CMainSignals; friend class ValidationInterfaceTest; @@ -199,9 +202,9 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef&, uint64_t mempool_sequence); void TransactionRemovedFromMempool(const CTransactionRef&, MemPoolRemovalReason, uint64_t mempool_sequence); - void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex); + void BlockConnected(ChainstateRole, const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex); void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); - void ChainStateFlushed(const CBlockLocator &); + void ChainStateFlushed(ChainstateRole, const CBlockLocator &); void BlockChecked(const CBlock&, const BlockValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 512a011dfc..f4cb4bbd66 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -162,11 +162,11 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) } Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<bilingual_str>& errors, - CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx, bool require_mine, const std::vector<CTxOut>& outputs, std::optional<uint32_t> reduce_output) + CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx, bool require_mine, const std::vector<CTxOut>& outputs, std::optional<uint32_t> original_change_index) { - // Cannot both specify new outputs and an output to reduce - if (!outputs.empty() && reduce_output.has_value()) { - errors.push_back(Untranslated("Cannot specify both new outputs to use and an output index to reduce")); + // For now, cannot specify both new outputs to use and an output index to send change + if (!outputs.empty() && original_change_index.has_value()) { + errors.push_back(Untranslated("The options 'outputs' and 'original_change_index' are incompatible. You can only either specify a new set of outputs, or designate a change output to be recycled.")); return Result::INVALID_PARAMETER; } @@ -182,8 +182,8 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } const CWalletTx& wtx = it->second; - // Make sure that reduce_output is valid - if (reduce_output.has_value() && reduce_output.value() >= wtx.tx->vout.size()) { + // Make sure that original_change_index is valid + if (original_change_index.has_value() && original_change_index.value() >= wtx.tx->vout.size()) { errors.push_back(Untranslated("Change position is out of range")); return Result::INVALID_PARAMETER; } @@ -259,7 +259,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo const CTxOut& output = txouts.at(i); CTxDestination dest; ExtractDestination(output.scriptPubKey, dest); - if (reduce_output.has_value() ? reduce_output.value() == i : OutputIsChange(wallet, output)) { + if (original_change_index.has_value() ? original_change_index.value() == i : OutputIsChange(wallet, output)) { new_coin_control.destChange = dest; } else { CRecipient recipient = {dest, output.nValue, false}; diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index f00bf15730..d3d43861ef 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -44,7 +44,7 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid); * @param[out] mtx The bump transaction itself * @param[in] require_mine Whether the original transaction must consist of inputs that can be spent by the wallet * @param[in] outputs Vector of new outputs to replace the bumped transaction's outputs - * @param[in] reduce_output The position of the change output to deduct the fee from in the transaction being bumped + * @param[in] original_change_index The position of the change output to deduct the fee from in the transaction being bumped */ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, @@ -55,7 +55,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, CMutableTransaction& mtx, bool require_mine, const std::vector<CTxOut>& outputs, - std::optional<uint32_t> reduce_output = std::nullopt); + std::optional<uint32_t> original_change_index = std::nullopt); //! Sign the new transaction, //! @return false if the tx couldn't be found or if it was diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index c2f4321a60..6b96fc4e49 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1016,10 +1016,14 @@ static RPCHelpMan bumpfee_helper(std::string method_name) {"outputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "The outputs specified as key-value pairs.\n" "Each key may only appear once, i.e. there can only be one 'data' output, and no address may be duplicated.\n" "At least one output of either type must be specified.\n" - "Cannot be provided if 'reduce_output' is specified.", + "Cannot be provided if 'original_change_index' is specified.", OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, - {"reduce_output", RPCArg::Type::NUM, RPCArg::DefaultHint{"not set, detect change automatically"}, "The 0-based index of the output from which the additional fees will be deducted. In general, this should be the position of change output. Cannot be provided if 'outputs' is specified."}, + {"original_change_index", RPCArg::Type::NUM, RPCArg::DefaultHint{"not set, detect change automatically"}, "The 0-based index of the change output on the original transaction. " + "The indicated output will be recycled into the new change output on the bumped transaction. " + "The remainder after paying the recipients and fees will be sent to the output script of the " + "original change output. The change output’s amount can increase if bumping the transaction " + "adds new inputs, otherwise it will decrease. Cannot be used in combination with the 'outputs' option."}, }, RPCArgOptions{.oneline_description="options"}}, }, @@ -1058,7 +1062,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) coin_control.m_signal_bip125_rbf = true; std::vector<CTxOut> outputs; - std::optional<uint32_t> reduce_output; + std::optional<uint32_t> original_change_index; if (!request.params[1].isNull()) { UniValue options = request.params[1]; @@ -1070,7 +1074,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) {"replaceable", UniValueType(UniValue::VBOOL)}, {"estimate_mode", UniValueType(UniValue::VSTR)}, {"outputs", UniValueType()}, // will be checked by AddOutputs() - {"reduce_output", UniValueType(UniValue::VNUM)}, + {"original_change_index", UniValueType(UniValue::VNUM)}, }, true, true); @@ -1095,8 +1099,8 @@ static RPCHelpMan bumpfee_helper(std::string method_name) outputs = tempTx.vout; } - if (options.exists("reduce_output")) { - reduce_output = options["reduce_output"].getInt<uint32_t>(); + if (options.exists("original_change_index")) { + original_change_index = options["original_change_index"].getInt<uint32_t>(); } } @@ -1115,7 +1119,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) CMutableTransaction mtx; feebumper::Result res; // Targeting feerate bump. - res = feebumper::CreateRateBumpTransaction(*pwallet, hash, coin_control, errors, old_fee, new_fee, mtx, /*require_mine=*/ !want_psbt, outputs, reduce_output); + res = feebumper::CreateRateBumpTransaction(*pwallet, hash, coin_control, errors, old_fee, new_fee, mtx, /*require_mine=*/ !want_psbt, outputs, original_change_index); if (res != feebumper::Result::OK) { switch(res) { case feebumper::Result::INVALID_ADDRESS_OR_KEY: diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 9d5ce6e125..7c0eca1475 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -123,20 +123,14 @@ public: template<typename Stream> void Serialize(Stream& s) const { - int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) { - s << nVersion; - } + s << int{259900}; // Unused field, writes the highest client version ever written s << nTime << vchPubKey << fInternal << m_pre_split; } template<typename Stream> void Unserialize(Stream& s) { - int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) { - s >> nVersion; - } + s >> int{}; // Discard unused field s >> nTime >> vchPubKey; try { s >> fInternal; diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 42accafe5b..abd788f96f 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <kernel/chain.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -145,8 +146,8 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) // time to the maximum value. This ensures that the wallet's birth time is always // earlier than this maximum time. info.chain_time_max = std::numeric_limits<unsigned int>::max(); - a.wallet->blockConnected(info); - b.wallet->blockConnected(info); + a.wallet->blockConnected(ChainstateRole::NORMAL, info); + b.wallet->blockConnected(ChainstateRole::NORMAL, info); // Store the coins for the next block Coins coins_new; for (const auto& tx : block.vtx) { diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index dac6e87983..21ed52731a 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -752,14 +752,14 @@ bool malformed_descriptor(std::ios_base::failure e) BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) { std::vector<unsigned char> malformed_record; - CVectorWriter vw(0, 0, malformed_record, 0); + CVectorWriter vw{0, malformed_record, 0}; vw << std::string("notadescriptor"); vw << uint64_t{0}; vw << int32_t{0}; vw << int32_t{0}; vw << int32_t{1}; - SpanReader vr{0, 0, malformed_record}; + SpanReader vr{0, malformed_record}; WalletDescriptor w_desc; BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 2459908419..c240e88531 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -22,6 +22,7 @@ #include <interfaces/chain.h> #include <interfaces/handler.h> #include <interfaces/wallet.h> +#include <kernel/chain.h> #include <kernel/mempool_removal_reason.h> #include <key.h> #include <key_io.h> @@ -626,11 +627,11 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; } -void CWallet::chainStateFlushed(const CBlockLocator& loc) +void CWallet::chainStateFlushed(ChainstateRole role, const CBlockLocator& loc) { // Don't update the best block until the chain is attached so that in case of a shutdown, // the rescan will be restarted at next startup. - if (m_attaching_chain) { + if (m_attaching_chain || role == ChainstateRole::BACKGROUND) { return; } WalletBatch batch(GetDatabase()); @@ -1339,11 +1340,14 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c { LOCK(cs_wallet); - int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; // If number of conflict confirms cannot be determined, this means // that the block is still unknown or not yet part of the main chain, // for example when loading the wallet during a reindex. Do nothing in that // case. + if (m_last_block_processed_height < 0 || conflicting_height < 0) { + return; + } + int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; if (conflictconfirms >= 0) return; @@ -1462,8 +1466,11 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe } } -void CWallet::blockConnected(const interfaces::BlockInfo& block) +void CWallet::blockConnected(ChainstateRole role, const interfaces::BlockInfo& block) { + if (role == ChainstateRole::BACKGROUND) { + return; + } assert(block.data); LOCK(cs_wallet); @@ -2941,7 +2948,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } if (chain) { - walletInstance->chainStateFlushed(chain->getTipLocator()); + walletInstance->chainStateFlushed(ChainstateRole::NORMAL, chain->getTipLocator()); } } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation @@ -3227,7 +3234,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf } } walletInstance->m_attaching_chain = false; - walletInstance->chainStateFlushed(chain.getTipLocator()); + walletInstance->chainStateFlushed(ChainstateRole::NORMAL, chain.getTipLocator()); walletInstance->GetDatabase().IncrementUpdateCounter(); } walletInstance->m_attaching_chain = false; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5adb8b6e27..9333493a6e 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -599,7 +599,7 @@ public: CWalletTx* AddToWallet(CTransactionRef tx, const TxState& state, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx) override; - void blockConnected(const interfaces::BlockInfo& block) override; + void blockConnected(ChainstateRole role, const interfaces::BlockInfo& block) override; void blockDisconnected(const interfaces::BlockInfo& block) override; void updatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); @@ -777,7 +777,7 @@ public: /** should probably be renamed to IsRelevantToMe */ bool IsFromMe(const CTransaction& tx) const; CAmount GetDebit(const CTransaction& tx, const isminefilter& filter) const; - void chainStateFlushed(const CBlockLocator& loc) override; + void chainStateFlushed(ChainstateRole role, const CBlockLocator& loc) override; DBErrors LoadWallet(); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 6755368249..03aae86577 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -5,6 +5,7 @@ #include <zmq/zmqnotificationinterface.h> #include <common/args.h> +#include <kernel/chain.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -170,8 +171,11 @@ void CZMQNotificationInterface::TransactionRemovedFromMempool(const CTransaction }); } -void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) +void CZMQNotificationInterface::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) { + if (role == ChainstateRole::BACKGROUND) { + return; + } for (const CTransactionRef& ptx : pblock->vtx) { const CTransaction& tx = *ptx; TryForEachAndRemoveFailed(notifiers, [&tx](CZMQAbstractNotifier* notifier) { diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index ce67633b30..4246c53bd3 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -33,7 +33,7 @@ protected: // CValidationInterface void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; + void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; |